diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 151a5fd..dddd0f4 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -67,3 +67,21 @@ jobs: uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} + + # cliffy has to work, lint and pass its tests without the optional `rich` and `ai` extras + no-extras: + name: Without extras (Python 3.11) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install uv + uses: astral-sh/setup-uv@v10.3.0 + with: + python-version: "3.11" + enable-cache: true + - name: Install library without extras + run: uv sync --locked + - name: Code quality + run: make lint + - name: Test with pytest + run: make test diff --git a/Makefile b/Makefile index 4c56b0d..4c4f13f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ SOURCE_FILES=cliffy tests -GENERATE_DEPS=requests "six<1.0.0" keyring check: format lint test clean @@ -50,24 +49,17 @@ docs-check: docs-build: docs-check uv run --group docs mkdocs build --strict -# The example manifests declare requirements that are not cliffy dependencies. They have to be -# installed into the project environment (not a `uv run --with` overlay), because cliffy validates -# `requires` against `pip freeze`. `uv sync` in generate-cleanup removes them again. +# examples/generated/*.py are what cliffy generates for the manifests in examples/. Nothing is installed or +# loaded for this: the manifests are only rendered. tests/test_examples.py fails when a file is stale. generate-clis: - uv pip install ${GENERATE_DEPS} - uv run --no-sync cli load examples/*.yaml - cp cliffy/clis/*.py examples/generated/ + uv run python scripts/examples.py -generate-all: generate-clis generate-schema generate-cleanup +generate-all: generate-clis generate-schema @echo "~ done" -generate-cleanup: - uv run --no-sync cli rm-all - uv sync --all-extras --group docs - generate-schema: uv run python -m cliffy.manifest --json-schema > examples/cliffy_schema.json .PHONY: check install sync test test-cov clean package publish format lint \ docs-serve docs-generate docs-check docs-build \ - generate-clis generate-all generate-cleanup generate-schema + generate-clis generate-all generate-schema diff --git a/README.md b/README.md index 9adc0b2..9dfe52a 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ HELLO, ADA! | `cli test ` | Run the tests defined in the manifest | | `cli build ` | Build one file to ship: a zipapp, a script or a native executable | | `cli render ` | Print the generated Python code | -| `cli validate ` | Check a manifest against the schema | +| `cli validate ` | Check a manifest and the code generated from it | | `cli docs ` | Generate documentation for the CLI | | `cli list`, `info`, `update`, `remove` | Manage loaded CLIs | | `cli ai generate`, `cli ai ask` | Draft manifests with an LLM (needs the `ai` extra) | @@ -93,7 +93,7 @@ Every command and option is described in the ```console $ pip install cliffy # core -$ pip install "cliffy[rich]" # rich-click help for the cli command, needed for the Click backend +$ pip install "cliffy[rich]" # rich-click help for the cli command and for Click-backend CLIs $ pip install "cliffy[ai]" # AI commands, built on pydantic-ai $ uvx cliffy run hello.yaml greet Ada # run without installing ``` diff --git a/cliffy/ai.py b/cliffy/ai.py index 84732b4..4627625 100644 --- a/cliffy/ai.py +++ b/cliffy/ai.py @@ -278,7 +278,6 @@ def generate_manifest( ) rejections: list[ManifestOutputError] = [] - @agent.output_validator def validate_output(output: str) -> str: try: return validate_manifest_yaml(output, cli_name) @@ -286,6 +285,10 @@ def validate_output(output: str) -> str: rejections.append(e) raise ModelRetry(str(e)) from e + # registered with a call, not as a decorator: without the `ai` extra the type checker cannot see + # pydantic-ai, and an untyped decorator would make this function untyped + agent.output_validator(validate_output) + try: result = agent.run_sync(description) except UnexpectedModelBehavior as e: diff --git a/cliffy/blueprint.py b/cliffy/blueprint.py index c3ae20f..7d7f0ae 100644 --- a/cliffy/blueprint.py +++ b/cliffy/blueprint.py @@ -130,6 +130,12 @@ def normalize_reference(reference: str, base_dir: str | os.PathLike[str] | None return f"{resolved}:{attr}" if attr else resolved +def blueprint_file(reference: str, base_dir: str | os.PathLike[str] | None = None) -> str | None: + """Absolute path of the Python file a reference points to, or None when it names a module or a plugin.""" + target, _ = _split_reference(normalize_reference(reference, base_dir)) + return target if _is_file_reference(target) else None + + def resolve_blueprint(reference: str, base_dir: str | os.PathLike[str] | None = None) -> type[Blueprint]: """Resolve a blueprint reference to a ``Blueprint`` subclass. diff --git a/cliffy/builder.py b/cliffy/builder.py index 78c95e0..928a873 100644 --- a/cliffy/builder.py +++ b/cliffy/builder.py @@ -34,7 +34,12 @@ MIN_PYTHON = ">=3.11" # Package that provides the framework a generated CLI imports FRAMEWORK_PACKAGES = {"typer": "typer", "rich_click": "rich-click", "click": "click"} -GENERATED_HEADER = re.compile(r"\A## Generated (\S+) on .*\n") +# An import of a framework at the start of a line, at any indentation: generated Click code imports +# rich_click in a `try` block and falls back to click in the `except` block. +FRAMEWORK_IMPORT = re.compile(rf"^[ \t]*(?:import|from)[ \t]+({'|'.join(FRAMEWORK_PACKAGES)})\b", re.MULTILINE) +# Header of code generated by cliffy 0.6 and older, which holds the time of generation. A CLI loaded with +# such a version is still built from its script. Current headers name the cliffy version and are kept. +LEGACY_GENERATED_HEADER = re.compile(r"\A## Generated (\S+) on .*\n") # Release of PyApp (https://github.com/ofek/pyapp), the launcher a native build compiles PYAPP_VERSION = "0.30.0" INSTALL_RUST = ( @@ -364,12 +369,12 @@ def build_cli( def get_standalone_code(code: str) -> str: - """Generated code as it is shipped: no timestamp, and no import of pybash. + """Generated code as it is shipped: no timestamp in the header, and no import of pybash. A CLI with shell operators calls a helper of pybash. Its source is pasted in where the import was, so that a built CLI depends on its framework and never on pybash or cliffy. """ - code = GENERATED_HEADER.sub(r"## Generated \1 with cliffy\n", code) + code = LEGACY_GENERATED_HEADER.sub(r"## Generated \1 with cliffy\n", code) return code.replace(SHELL_RUNTIME_IMPORT, f"\n\n{pybash.runtime_source()}\n\n", 1) @@ -472,9 +477,16 @@ def get_build_requirements(code: str, deps: list[str]) -> list[str]: with contextlib.suppress(InvalidRequirement): named.add(canonicalize_name(Requirement(dep).name)) + imported = set(FRAMEWORK_IMPORT.findall(code)) + if imported & {"typer", "rich_click"}: + # Typer and rich-click depend on Click and bring a version they work with. Click is only a + # requirement of its own for a CLI that imports nothing else. This is also the generated Click + # code, which prefers rich_click and falls back to click: the build ships rich-click. + imported.discard("click") + requirements = list(deps) for module, package in FRAMEWORK_PACKAGES.items(): - if package in named or not re.search(rf"^\s*(import|from) {module}\b", code, flags=re.MULTILINE): + if package in named or module not in imported: continue try: requirements.append(f"{package}=={metadata.version(package)}") diff --git a/cliffy/checker.py b/cliffy/checker.py new file mode 100644 index 0000000..25bb83c --- /dev/null +++ b/cliffy/checker.py @@ -0,0 +1,75 @@ +"""Checks the code generated for a manifest and reports problems in the manifest's terms.""" + +from __future__ import annotations + +import re +import traceback + +import typer + +from cliffy.commander import CLI + +GENERATED_FILE = "" +FUNCTION_DEFINITION = re.compile(r"(?:async\s+)?def (\w+)\(") +MODULE_LEVEL = "module level code (`imports`, `vars`, `types` or `functions`)" + + +class GeneratedCodeError(Exception): + """The code generated for a manifest does not compile or does not import. + + The message names the part of the manifest the failing code was generated from. + """ + + +def locate(cli: CLI, line_number: int) -> str: + """Part of the manifest that line `line_number` (1-based) of the generated code was generated from.""" + lines = cli.code.splitlines() + index = min(max(line_number, 1), len(lines)) - 1 + + # a decorator belongs to the function below it + while index < len(lines) - 1 and lines[index].startswith("@"): + index += 1 + + for line in reversed(lines[: index + 1]): + if match := FUNCTION_DEFINITION.match(line): + return cli.functions.get(match.group(1), MODULE_LEVEL) + return MODULE_LEVEL + + +def describe(cli: CLI, error: BaseException, line_number: int | None) -> str: + where = locate(cli, line_number) if line_number else MODULE_LEVEL + message = f"{where}: {type(error).__name__}: {error.msg if isinstance(error, SyntaxError) else error}" + lines = cli.code.splitlines() + if line_number and 0 < line_number <= len(lines): + message += f"\n {lines[line_number - 1].strip()}" + return message + + +def check_generated_code(cli: CLI) -> None: + """Compiles and imports the generated code, and builds the command tree of a Typer CLI. + + Importing defines the commands, it does not run them. + + Raises: + GeneratedCodeError: the code is not valid Python, or fails while it is imported. + """ + try: + # dont_inherit: this module's own `from __future__ import annotations` must not apply to the generated code + compiled = compile(cli.code, GENERATED_FILE, "exec", dont_inherit=True) + except SyntaxError as e: + raise GeneratedCodeError(describe(cli, e, e.lineno)) from e + + try: + namespace: dict[str, object] = {"__name__": f"cliffy_check_{cli.name.replace('-', '_')}"} + exec(compiled, namespace) + app = namespace.get("cli") + if isinstance(app, typer.Typer): + # Typer reads the signature of a command when the CLI starts. From Python 3.14 on annotations + # are evaluated lazily, so a param with an unknown type only fails here + typer.main.get_command(app) + except (Exception, SystemExit) as e: + generated_frames = [ + frame for frame in traceback.extract_tb(e.__traceback__) if frame.filename == GENERATED_FILE + ] + line_number = generated_frames[-1].lineno if generated_frames else None + raise GeneratedCodeError(describe(cli, e, line_number)) from e diff --git a/cliffy/cli.py b/cliffy/cli.py index abd9f53..d6dff7a 100644 --- a/cliffy/cli.py +++ b/cliffy/cli.py @@ -17,11 +17,13 @@ format_size, run_cli, ) +from cliffy.checker import GeneratedCodeError, check_generated_code from cliffy.doc import DocGenerator from cliffy.helper import ( CLI_NAME_PATTERN, ManifestOrCLI, age_datetime, + cliffy_version, exit_err, indent_block, out, @@ -67,7 +69,8 @@ def show_aliases_callback(ctx: Any, param: Any, val: bool) -> None: @click.group(context_settings=CONTEXT_SETTINGS) -@click.version_option() +# the version is passed in: with rich-click installed, the automatic lookup finds rich-click's own version +@click.version_option(cliffy_version(), package_name="cliffy") @click.option("--aliases", type=bool, is_flag=True, is_eager=True, callback=show_aliases_callback) def cli(aliases: bool) -> None: pass @@ -340,14 +343,18 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None: out("✨ Invoking tests ✨", nl=False) total = tester.total_cases passed = 0 - for i, case in enumerate(tester.test_pipeline): + case_number = 0 + for case in tester.test_pipeline: if isinstance(case, ShellScript): + # a setup or cleanup step: it is run, but it is not one of the counted test cases try: tester.invoke_shell(case) except SyntaxError as e: exit_err(f"~ error in {manifest}: test step `{case.command}`, column {e.offset}: {e.msg}") continue + case_number += 1 + out(f"\n\n🪄 > {tester.T.cli.name} {case.command}\n") try: test = tester.invoke_test(case.command, case.assert_script) @@ -356,7 +363,7 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None: if result.exception: out(str(result)) next(test, "") - out(f"\n✅ {i + 1} of {total}") + out(f"\n✅ {case_number} of {total}") passed += 1 except AssertionError: _, _, tb = sys.exc_info() @@ -390,12 +397,15 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None: @click.argument("manifest", type=click.File("rb"), required=True) @blueprint_option def validate(manifest: TextIO, blueprint: str | None = None) -> None: - """Validate the syntax and structure of a CLI manifest""" + """Validate a CLI manifest: its structure, its requirements and the code generated from it""" try: - Transformer(manifest, blueprint=blueprint) - out(f"Manifest {manifest.name} is valid", fg="green") + T = Transformer(manifest, blueprint=blueprint) + check_generated_code(T.cli) + except GeneratedCodeError as e: + exit_err(f"Manifest {manifest.name} is invalid: the generated code does not work\n {e}") except Exception as e: exit_err(f"Manifest {manifest.name} is invalid: {e}") + out(f"Manifest {manifest.name} is valid", fg="green") @click.argument("cli_or_manifest", type=ManifestOrCLI()) diff --git a/cliffy/commander.py b/cliffy/commander.py index f198983..353b5bd 100644 --- a/cliffy/commander.py +++ b/cliffy/commander.py @@ -1,14 +1,16 @@ from __future__ import annotations +import ast import re from abc import ABC, abstractmethod from collections import defaultdict from collections.abc import ItemsView, Iterator, ValuesView +from contextlib import contextmanager import pybash from pydantic import BaseModel -from cliffy.helper import py_str, to_identifier +from cliffy.helper import cliffy_version, py_str, to_identifier from cliffy.manifest import ( Callback, CLIManifest, @@ -23,7 +25,7 @@ RunBlockList, SimpleCommandParam, ) -from cliffy.parser import Parser, transform_script +from cliffy.parser import Parser, ScriptError, transform_script # The helper behind shell lines with operators (`|`, `&&`, `>` ...) or with values quoted for a `>` line. # Plain `$ command` lines are bare `subprocess` calls and do not use it. @@ -42,6 +44,8 @@ class CLI(BaseModel): code: str requires: list[str] = [] blueprint: str | None = None + # generated function name -> where it comes from in the manifest, i.e. {"db_create": "commands.db.create"} + functions: dict[str, str] = {} class BaseGroup(BaseModel): @@ -50,11 +54,13 @@ class BaseGroup(BaseModel): parent_group: BaseGroup | None = None commands: list[Command] = [] help: str = "" + # set by the Commander, which knows the other names of the generated module + var: str = "" @property def var_name(self) -> str: """Returns valid Python variable name for group app""" - return to_identifier(self.name + "_app") + return self.var or to_identifier(self.name + "_app") class Groups(BaseModel): @@ -112,13 +118,16 @@ class Commander(ABC): "base_imports", "callbacks_by_group", "cli", + "command_functions", "commands", + "functions", "greedy", "groups", "manifest", "parser", "root_commands", "root_group", + "taken_names", ) def __init__(self, manifest: CLIManifest) -> None: @@ -148,6 +157,57 @@ def __init__(self, manifest: CLIManifest) -> None: ] self.build_groups() + # Python names of the generated module. Every command function and group variable gets one that + # nothing else uses: two manifest names can map to the same identifier (`db-app` and the group `db`). + self.taken_names: set[str] = set(self.manifest.vars) | self.user_defined_names() + self.functions: dict[str, str] = {} + self.command_functions: dict[str, str] = {} + for group in self.groups.values(): + group.var = self.unique_name(to_identifier(group.name + "_app")) + + def user_defined_names(self) -> set[str]: + """Names bound by the `imports` and `functions` sections, as far as they can be read as Python.""" + names: set[str] = set() + for section in (self.manifest.imports, self.manifest.functions): + for block in [section] if isinstance(section, str) else section: + try: + tree = ast.parse(block) + except SyntaxError: + # i.e. a function with shell lines. Its name cannot be known before it is transformed + continue + for node in tree.body: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): + names.add(node.name) + elif isinstance(node, ast.Import | ast.ImportFrom): + names.update((alias.asname or alias.name).split(".")[0] for alias in node.names) + elif isinstance(node, ast.Assign): + names.update(target.id for target in node.targets if isinstance(target, ast.Name)) + return names + + def unique_name(self, name: str, source: str = "") -> str: + """Reserves a Python name for the generated module: `name`, or `name_2`, `name_3`, ... when it is taken. + + Args: + name (str): The identifier to reserve. + source (str): Where a function with this name comes from in the manifest, i.e. `commands.db.create`. + """ + unique, count = name, 1 + while unique in self.taken_names: + count += 1 + unique = f"{name}_{count}" + self.taken_names.add(unique) + if source: + self.functions[unique] = source + return unique + + def command_func_name(self, command: Command) -> str: + """Name of the Python function generated for a command""" + if command.name not in self.command_functions: + self.command_functions[command.name] = self.unique_name( + self.parser.get_command_func_name(command), source=f"command `{command.name}`" + ) + return self.command_functions[command.name] + def _merge_command_template(self, command: Command) -> None: """Merge command with its template if specified.""" if not command.template: @@ -155,7 +215,9 @@ def _merge_command_template(self, command: Command) -> None: template = self.manifest.command_templates.get(command.template) if not template: - raise ValueError(f"Template {command.template} undefined in command_templates") + raise ValueError( + f"command `{command.name}`: template `{command.template}` is not defined in `command_templates`" + ) if template.params: command.params = template.params + (command.params or []) @@ -213,7 +275,19 @@ def add_subcommands(self) -> None: for group in self.groups.values(): self.add_group(group) for subcommand in group.commands: - self.add_sub_command(subcommand, group) + with self.reported_against(subcommand): + self.add_sub_command(subcommand, group) + + @contextmanager + def reported_against(self, command: Command) -> Iterator[None]: + """Names the command in the manifest that a code generation error belongs to""" + try: + yield + except ScriptError: + # an error in a script already says which command and which of its blocks it is in + raise + except ValueError as e: + raise ValueError(f"command `{command.name}`: {e}") from e def generate_cli(self) -> None: self.add_base_imports() @@ -231,9 +305,15 @@ def generate_cli(self) -> None: if SHELL_RUNTIME_REFERENCE.search(self.cli, base_imports_end): self.cli = self.cli[:base_imports_end] + SHELL_RUNTIME_IMPORT + self.cli[base_imports_end:] + def header(self) -> str: + """First line of the generated code. It holds nothing that changes between two runs, such as the time: + the same manifest gives the same code.""" + return f"## Generated {self.manifest.name} with cliffy {cliffy_version()}\n" + def add_root_commands(self) -> None: for root_command in self.root_group.commands: - self.add_root_command(root_command) + with self.reported_against(root_command): + self.add_root_command(root_command) def add_imports(self) -> None: if not self.manifest.imports: @@ -254,8 +334,11 @@ def add_vars(self) -> None: if isinstance(val, dict): # `name: {expression: null}` is the documented way to assign a Python expression self.cli += f"{var} = {next(iter(val))}\n" - else: + elif isinstance(val, str): self.cli += f"{var} = {py_str(val)}\n" + else: + # a number or a boolean is written as the Python literal of the same value + self.cli += f"{var} = {val!r}\n" self.cli += "\n" def add_functions(self) -> None: @@ -271,7 +354,8 @@ def add_functions(self) -> None: def add_command(self, command: Command) -> None: if "." in command.name: group = command.name.rsplit(".", 1)[0] - self.add_sub_command(command, self.groups[group]) + with self.reported_against(command): + self.add_sub_command(command, self.groups[group]) def add_greedy_commands(self) -> None: """Greedy commands get lazy-loaded. Only supported for group-commands currently""" @@ -402,4 +486,10 @@ def generate_cli(manifest: CLIManifest, commander_cls: type[Commander] = Command """ commander = commander_cls(manifest) commander.generate_cli() - return CLI(name=manifest.name, version=manifest.version, code=commander.cli, requires=manifest.requires) + return CLI( + name=manifest.name, + version=manifest.version, + code=commander.cli, + requires=manifest.requires, + functions=commander.functions, + ) diff --git a/cliffy/commanders/click.py b/cliffy/commanders/click.py index 768e34f..b9c91e1 100644 --- a/cliffy/commanders/click.py +++ b/cliffy/commanders/click.py @@ -1,9 +1,8 @@ import ast -import datetime import re from cliffy.commander import BaseGroup, Commander, callback_label -from cliffy.helper import py_str +from cliffy.helper import clean_help, py_str from cliffy.manifest import ( Callback, CLIManifest, @@ -22,15 +21,37 @@ class ClickCommander(Commander): def __init__(self, manifest: CLIManifest) -> None: super().__init__(manifest) self.base_imports.add("import subprocess") - self.base_imports.add("import rich_click as click") self.base_imports.add("from typing import Optional, Any") self.click_parser = ClickParser(manifest) def add_base_imports(self) -> None: - self.cli = f"""## Generated {self.manifest.name} on {datetime.datetime.now()}\n""" + self.cli = self.header() # sorted: a set has no stable order, which made the generated code differ from run to run for imp in sorted(self.base_imports): self.cli += imp + "\n" + self.cli += self.click_import() + + def click_import(self) -> str: + """Import of `click`: rich-click when it is installed, plain Click otherwise. + + rich-click is an optional dependency (`cliffy[rich]`), Click is not. `cli build` reads this import + and bundles rich-click. A manifest that asks for a rich-click feature gets the import without a + fallback. + """ + if self.uses_rich_click_features(): + return "import rich_click as click\n" + return ( + "try:\n" + " import rich_click as click\n" + "except ImportError: # plain help output when rich-click is not installed\n" + " import click\n" + ) + + def uses_rich_click_features(self) -> bool: + """Whether a command is placed in a help panel, which only rich-click can do""" + configs = [command.config for command in self.commands] + configs += [template.config for template in self.manifest.command_templates.values()] + return any(config is not None and bool(config.rich_help_panel) for config in configs) def add_base_cli(self) -> None: self.cli += """ @@ -45,6 +66,7 @@ def show_aliases(ctx, param, value): ctx.exit() """ callbacks = self.callbacks_for() + self.functions["cli"] = "callback of the CLI" self.cli += """ @click.group(context_settings=CONTEXT_SETTINGS""" if "invoke_without_command" not in self.manifest.cli_options: @@ -92,8 +114,9 @@ def parse_callback_run(self, callbacks: list[Callback]) -> str: def define_groups(self) -> None: for group in self.groups.values(): - parsed_help = group.help.replace("\n", "") if group.help else "" + parsed_help = clean_help(group.help) callbacks = self.callbacks_for(group) + self.functions[group.var_name] = f"callback of group `{group.name}`" self.cli += f""" @click.group({self.parse_callback_options(callbacks)}){self.parse_callback_decorators(callbacks)} def {group.var_name}({self.parse_callback_param_names(callbacks)}): @@ -110,12 +133,12 @@ def add_root_command(self, command: Command) -> None: if not command.run: return - parsed_command_func_name = self.click_parser.get_command_func_name(command) + parsed_command_func_name = self.command_func_name(command) parsed_command_name = self.click_parser.get_parsed_command_name(command) - parsed_help = command.help.replace("\n", "") if command.help else "" + parsed_help = clean_help(command.help) self.cli += f""" -@cli.command(name={py_str(parsed_command_name)}) +@cli.command(name={py_str(parsed_command_name)}{self.click_parser.get_parsed_config(command)}) {self.click_parser.parse_params(command)} def {parsed_command_func_name}({self.click_parser.get_param_names(command)}): {py_str(parsed_help)} @@ -126,18 +149,22 @@ def {parsed_command_func_name}({self.click_parser.get_param_names(command)}): self.cli += f""" @cli.command(name={py_str(alias)}, hidden=True) {self.click_parser.parse_params(command)} -def {self.click_parser.get_alias_func_name(command, alias)}({self.click_parser.get_param_names(command)}): +def {self.alias_func_name(command, alias)}({self.click_parser.get_param_names(command)}): {py_str("Alias for " + parsed_command_name)} {self.click_parser.parse_command_run(command)} """ def add_sub_command(self, command: Command, group: BaseGroup) -> None: - parsed_command_func_name = self.click_parser.get_command_func_name(command) + if not command.run: + # a script-less command only carries the help of the group with the same name + return + + parsed_command_func_name = self.command_func_name(command) parsed_command_name = self.click_parser.get_parsed_command_name(command) - parsed_help = command.help.replace("\n", "") if command.help else "" + parsed_help = clean_help(command.help) self.cli += f""" -@{group.var_name}.command(name={py_str(parsed_command_name)}) +@{group.var_name}.command(name={py_str(parsed_command_name)}{self.click_parser.get_parsed_config(command)}) {self.click_parser.parse_params(command)} def {parsed_command_func_name}({self.click_parser.get_param_names(command)}): {py_str(parsed_help)} @@ -148,11 +175,18 @@ def {parsed_command_func_name}({self.click_parser.get_param_names(command)}): self.cli += f""" @{group.var_name}.command(name={py_str(alias)}, hidden=True) {self.click_parser.parse_params(command)} -def {self.click_parser.get_alias_func_name(command, alias)}({self.click_parser.get_param_names(command)}): +def {self.alias_func_name(command, alias)}({self.click_parser.get_param_names(command)}): {py_str("Alias for " + parsed_command_name)} {self.click_parser.parse_command_run(command)} """ + def alias_func_name(self, command: Command, alias: str) -> str: + """Name of the function that is registered under an alias: Click needs one function per command name.""" + return self.unique_name( + self.click_parser.get_alias_func_name(self.command_func_name(command), alias), + source=f"command `{command.name}`", + ) + def add_main_block(self) -> None: self.cli += """ if __name__ == "__main__": @@ -194,20 +228,106 @@ def parse_params(self, command: Command) -> str: def parse_param_list(self, params: list[ParamBlock]) -> str: """Parse a list of parameters into Click decorators""" - decorators = [] - for param in params: - if isinstance(param, GenericCommandParam): - decorators.append(param.root.strip()) - elif isinstance(param, CommandParam): - decorators.append(self._build_click_option(param)) - elif isinstance(param, SimpleCommandParam): - decorators.append(self._parse_simple_param(param)) + return "\n".join(self.parse_param(param) for param in params) + + def parse_param(self, param: ParamBlock) -> str: # type: ignore[override] + """Parse one param into a Click decorator""" + if isinstance(param, GenericCommandParam): + decorator = param.root.strip() + if not decorator.startswith("@"): + raise ValueError( + f"param `{decorator}` is a function signature entry, which only the Typer backend can use. " + "With `use_click: true` a string param has to be a Click decorator, " + "i.e. \"@click.option('--name', ...)\"." + ) + return decorator + + if isinstance(param, SimpleCommandParam): + if "typer." in param.raw_type: + raise ValueError( + f"param `{param.name}` is a Typer definition (`{param.raw_type.strip()}`), which the Click " + "backend cannot use. Write it as a Click decorator string, " + "i.e. \"@click.option('--name', ...)\", or remove `use_click`." + ) + default, required, help_text = param.default, param.required, "" + else: + default, required, help_text = param.default, param.required, param.help + if default is not None: + default = str(default) + + param_type = param.type.strip() + if param_type in self.manifest.types: + definition = self.manifest.types[param_type].strip() + if definition.startswith("@click."): + # a complete Click decorator: used as written, like a Typer definition is for the Typer backend + return definition + if "typer." in definition: + raise ValueError( + f"param `{param.name}` uses types.{param_type}, which is a Typer definition (`{definition}`). " + f"With `use_click: true` a type is a plain type such as `{param_type}: int`, or a Click decorator " + f"such as `{param_type}: \"@click.option('--name', ...)\"`." + ) + # `N: int = 5` names a type and a default. A default on the param itself wins. + param_type, _, type_default = (part.strip() for part in definition.partition("=")) + if default is None and type_default: + default = type_default + + click_type, multiple = self._click_type(param_type) + if default is not None: + if isinstance(param, CommandParam): + # a str default of an explicit param is data, not an expression: emit it as a string literal + default = repr(default) if click_type == "str" else default + else: + default = self._simple_default(click_type, default) + + self.normalize_param_name(param.name) # validates + if param.is_option(): + parts = self._option_declarations(param.name, param.short, click_type) + if click_type and click_type != "bool": + parts.append(f"type={click_type}") + if multiple: + parts.append("multiple=True") + if required: + parts.append("required=True") + if default is not None: + parts.append(f"default={default}") + if help_text: + parts.append(f"help={help_text!r}") + return f"@click.option({', '.join(parts)})" + + parts = [repr(param.name.strip())] + if click_type: + parts.append(f"type={click_type}") + if multiple: + parts.append("nargs=-1") + if required: + parts.append("required=True") + elif not required and default is None: + # an argument is optional unless it is marked with `!` or `required`, as in Typer + parts.append("required=False") + if default is not None: + parts.append(f"default={default}") + if help_text: + parts.append(f"help={help_text!r}") + return f"@click.argument({', '.join(parts)})" - return "\n".join(decorators) + OPTIONAL_TYPE = re.compile(r"(?:typing\.)?Optional\[(.+)\]|(.+?)\s*\|\s*None|None\s*\|\s*(.+)") + LIST_TYPE = re.compile(r"(?:typing\.)?[Ll]ist\[(.+)\]") + + def _click_type(self, param_type: str) -> tuple[str, bool]: + """Annotation -> (Click type, takes several values). Optional[int] -> int, list[str] -> str, several.""" + click_type = param_type.strip() + if match := self.OPTIONAL_TYPE.fullmatch(click_type): + click_type = next(group for group in match.groups() if group).strip() + multiple = False + if match := self.LIST_TYPE.fullmatch(click_type): + click_type, multiple = match.group(1).strip(), True + if click_type in ("Any", "typing.Any"): + click_type = "" + return click_type, multiple def _option_declarations(self, name: str, short: str | None, param_type: str) -> list[str]: """--dry-run with short d -> ['--dry-run', '-d']. A bool is a `--flag/--no-flag` switch, like in Typer.""" - self.normalize_param_name(name) # validates long_name = name.strip().lstrip("-") declaration = f"--{long_name}/--no-{long_name}" if param_type == "bool" else f"--{long_name}" declarations = [repr(declaration)] @@ -217,28 +337,7 @@ def _option_declarations(self, name: str, short: str | None, param_type: str) -> def _parse_simple_param(self, param: SimpleCommandParam) -> str: """Parse SimpleCommandParam into Click decorator""" - param_type = param.type - default_value = param.default - - if param.is_option(): - parts = self._option_declarations(param.name, param.short, param_type) - if param_type and param_type != "bool": - parts.append(f"type={param_type}") - if param.required: - parts.append("required=True") - if default_value is not None: - parts.append(f"default={self._simple_default(param_type, default_value)}") - - return f"@click.option({', '.join(parts)})" - - self.normalize_param_name(param.name) # validates - parts = [repr(param.name.strip())] - if param_type: - parts.append(f"type={param_type}") - if default_value is not None: - parts.append(f"default={self._simple_default(param_type, default_value)}") - - return f"@click.argument({', '.join(parts)})" + return self.parse_param(param) @staticmethod def _simple_default(param_type: str, default_value: str) -> str: @@ -252,26 +351,18 @@ def _simple_default(param_type: str, default_value: str) -> str: return repr(default_value) return repr(literal) if isinstance(literal, str) else repr(default_value) - def _build_click_option(self, param: CommandParam) -> str: - """Build Click option decorator from CommandParam""" - parts = self._option_declarations(param.name, param.short, param.type) - - if param.type and param.type != "bool": - parts.append(f"type={param.type}") - - if param.required: - parts.append("required=True") - - if param.default is not None: - # a str default is data, not an expression: emit it as a string literal - parts.append(f"default={str(param.default)!r}" if param.type == "str" else f"default={param.default}") - - if param.help: - parts.append(f"help={param.help!r}") + def get_parsed_config(self, command: Command) -> str: + """Command `config` as keyword arguments of Click's `command()`, with a leading comma when there are any.""" + if not command.config: + return "" - return f"@click.option({', '.join(parts)})" + configured_options = command.config.model_dump(exclude_unset=True) + if "rich_help_panel" in configured_options: + # rich-click calls the panel a command is listed under `panel` + configured_options["panel"] = configured_options.pop("rich_help_panel") + return f",{self.to_args(configured_options)}" if configured_options else "" - def get_alias_func_name(self, command: Command, alias: str) -> str: + def get_alias_func_name(self, command_func_name: str, alias: str) -> str: """Function name for the copy of a command that is registered under an alias""" alias_suffix = re.sub(r"\W", "_", alias) - return f"{self.get_command_func_name(command)}_{alias_suffix}" + return f"{command_func_name}_{alias_suffix}" diff --git a/cliffy/commanders/typer.py b/cliffy/commanders/typer.py index d92f4e1..1a15f2a 100644 --- a/cliffy/commanders/typer.py +++ b/cliffy/commanders/typer.py @@ -1,7 +1,5 @@ -import datetime - from cliffy.commander import BaseGroup, Commander, callback_label -from cliffy.helper import py_str +from cliffy.helper import clean_help, py_str from cliffy.manifest import Callback, CLIManifest, Command @@ -15,7 +13,7 @@ def __init__(self, manifest: CLIManifest) -> None: self.base_imports.add("from typing import Optional, Any") def add_base_imports(self) -> None: - self.cli = f"""## Generated {self.manifest.name} on {datetime.datetime.now()}\n""" + self.cli = self.header() # sorted: a set has no stable order, which made the generated code differ from run to run for imp in sorted(self.base_imports): self.cli += imp + "\n" @@ -49,6 +47,7 @@ def aliases_callback(value: bool): raise typer.Exit() """ callbacks = self.callbacks_for() + self.functions["main"] = "callback of the CLI" self.cli += f""" @cli.callback({self.parse_callback_options(callbacks)}) def main({self.parse_callback_params(callbacks)}""" @@ -89,10 +88,10 @@ def add_root_command(self, command: Command) -> None: if not command.run: return - parsed_command_func_name = self.parser.get_command_func_name(command) + parsed_command_func_name = self.command_func_name(command) parsed_command_name = self.parser.get_parsed_command_name(command) parsed_command_config = self.parser.get_parsed_config(command) - parsed_help = command.help.replace("\n", "") if command.help else "" + parsed_help = clean_help(command.help) empty_or_help = f"help={py_str(parsed_help)}," if parsed_help else "" command_args = f"{py_str(parsed_command_name)}, {empty_or_help}{parsed_command_config}" alias_epilog = py_str(f"Alias for {parsed_command_name}") @@ -111,14 +110,17 @@ def {parsed_command_func_name}({self.parser.parse_params(command)}): def define_groups(self) -> None: for group in self.groups.values(): - parsed_help = group.help.replace("\n", "") if group.help else "" + parsed_help = clean_help(group.help) empty_or_help = f"help={py_str(parsed_help)}," if parsed_help else "" self.cli += f"""{group.var_name} = typer.Typer({empty_or_help}) """ if callbacks := self.callbacks_for(group): + callback_name = self.unique_name( + f"{group.var_name}_callback", source=f"callback of group `{group.name}`" + ) self.cli += f""" @{group.var_name}.callback({self.parse_callback_options(callbacks)}) -def {group.var_name}_callback({self.parse_callback_params(callbacks)} +def {callback_name}({self.parse_callback_params(callbacks)} ): {self.parse_callback_run(callbacks)} """ @@ -126,7 +128,7 @@ def {group.var_name}_callback({self.parse_callback_params(callbacks)} def add_group(self, group: BaseGroup) -> None: """Add a group to the CLI with proper nesting""" parent_group = group.parent_group.var_name if group.parent_group else "cli" - parsed_help = group.help.replace("\n", "") if group.help else "" + parsed_help = clean_help(group.help) group_args = f"name={py_str(group.short_name)}, help={py_str(parsed_help)}" self.cli += f"""{parent_group}.add_typer({group.var_name}, {group_args}) """ @@ -137,10 +139,10 @@ def add_sub_command(self, command: Command, group: BaseGroup) -> None: # a script-less command only carries the help of the group with the same name return - parsed_command_func_name = self.parser.get_command_func_name(command) + parsed_command_func_name = self.command_func_name(command) parsed_command_name = self.parser.get_parsed_command_name(command) parsed_command_config = self.parser.get_parsed_config(command) - parsed_help = command.help.replace("\n", "") if command.help else "" + parsed_help = clean_help(command.help) empty_or_help = f"help={py_str(parsed_help)}," if parsed_help else "" command_args = f"{py_str(parsed_command_name)}, {empty_or_help}{parsed_command_config}" alias_epilog = py_str(f"Alias for {parsed_command_name}") diff --git a/cliffy/helper.py b/cliffy/helper.py index 0e97fff..b69a98e 100644 --- a/cliffy/helper.py +++ b/cliffy/helper.py @@ -7,7 +7,7 @@ import re import sys from datetime import datetime -from importlib.metadata import distributions +from importlib.metadata import PackageNotFoundError, distributions, version from importlib.resources import files from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -129,6 +129,16 @@ def py_str(value: object) -> str: return json.dumps(str(value), ensure_ascii=False) +def clean_help(text: str) -> str: + """Help text as one line per paragraph: a line break becomes a space, a blank line stays a paragraph break. + + Typer and Click reflow the lines of a paragraph and keep paragraphs apart, and both use the first + paragraph as the summary of a command in the command list. + """ + paragraphs = (" ".join(paragraph.split()) for paragraph in re.split(r"\n\s*\n", text or "")) + return "\n\n".join(paragraph for paragraph in paragraphs if paragraph) + + def to_identifier(name: str) -> str: """Turns name into a Python identifier that does not hide a keyword, a builtin or a generated name. @@ -144,6 +154,14 @@ def to_identifier(name: str) -> str: return identifier +def cliffy_version() -> str: + """Version of the installed cliffy package""" + try: + return version("cliffy") + except PackageNotFoundError: # pragma: no cover - cliffy is run from a source tree that is not installed + return "unknown" + + def get_installed_package_versions() -> dict[str, str]: """Installed distributions of the running interpreter, keyed by normalized (PEP 503) name.""" installed_packages: dict[str, str] = {} diff --git a/cliffy/homer.py b/cliffy/homer.py index 5b8a754..e2a5acf 100644 --- a/cliffy/homer.py +++ b/cliffy/homer.py @@ -70,7 +70,8 @@ def get_metadata_bypath(path: Path) -> CLIMetadata: return CLIMetadata(**json.load(metadata_file)) except Exception as e: return CLIMetadata( - cli_name=path.name, + # the file is .json: reported under that name, so that it can be removed with `cli rm` + cli_name=path.stem, runner_path=str(path.absolute()), version="error", loaded=datetime.now(), diff --git a/cliffy/manifest.py b/cliffy/manifest.py index 034072f..adb9f46 100644 --- a/cliffy/manifest.py +++ b/cliffy/manifest.py @@ -1,3 +1,5 @@ +import keyword +import re import sys from collections.abc import ItemsView, Iterator from datetime import datetime @@ -6,9 +8,30 @@ from pydantic import BaseModel, Field, RootModel, ValidationInfo, field_validator, model_validator -from .helper import CLI_NAME_PATTERN, wrap_as_comment +from .helper import CLI_NAME_PATTERN, GENERATED_NAMES, wrap_as_comment LATEST_SCHEMA_VERSION = "v3" +MIGRATION_GUIDE = "https://jaykv.github.io/cliffy/project/migration/" +# schema version -> the cliffy release that stopped reading it +DEPRECATED_SCHEMA_VERSIONS = {"v1": "0.4.0", "v2": "0.5.0"} + + +def manifest_version_problem(value: object) -> str | None: + """Why a `manifestVersion` value cannot be used and what to change, or None when it is the current version.""" + version = str(value).strip() + if version == LATEST_SCHEMA_VERSION: + return None + + use_latest = f"set `manifestVersion: {LATEST_SCHEMA_VERSION}` or remove the line" + if version in DEPRECATED_SCHEMA_VERSIONS: + return ( + f"The {version} schema is deprecated since cliffy {DEPRECATED_SCHEMA_VERSIONS[version]} and is no longer " + f"read. Update the manifest to the {LATEST_SCHEMA_VERSION} schema ({MIGRATION_GUIDE}), then {use_latest}." + ) + return ( + f"Unrecognized manifest version `{version}`. This version of cliffy reads `{LATEST_SCHEMA_VERSION}` " + f"manifests: {use_latest}." + ) class Example(BaseModel): @@ -116,7 +139,9 @@ def is_option(self) -> bool: ParamBlock = CommandParam | SimpleCommandParam | GenericCommandParam -VarBlock = str | dict[str, None] +# a string, a number or a boolean; or `{expression}` (a one-key mapping without a value) for a Python expression. +# bool is listed before int: in Python a bool is an int, and `true` has to stay a boolean. +VarBlock = str | bool | int | float | dict[str, None] class CommandConfig(BaseModel): @@ -124,10 +149,8 @@ class CommandConfig(BaseModel): context_settings: dict[Any, Any] = Field( default={}, - description="""Arbitrary settings passed to Click's context. Useful for things - like overriding the default `max_content_width`. - See Click's documentation for more details: - https://click.palletsprojects.com/en/8.1.x/advanced/#context-settings""", + description="Settings passed to the Click context of the command, for example `help_option_names` " + "or `max_content_width`.", ) epilog: str = Field(default="", description="Text displayed after the help message.") short_help: str = Field(default="", description="Short one-line help message displayed in help overviews.") @@ -147,9 +170,7 @@ class CommandConfig(BaseModel): ) rich_help_panel: str = Field( default="", - description="""Name of a Rich help panel to display after the default help. This is useful for - displaying more complex help information, such as tables or formatted text. - The content of the panel is defined using the `@rich_help` decorator.""", + description="Name of the panel this command is listed under in rich help output.", ) @@ -198,7 +219,11 @@ class Command(BaseModel): help: str = Field(default="", description="A description of the command, displayed in the help output.") params: list[ParamBlock] = Field( default=[], - description="A list of parameters for the command.\nThere are three ways to define a param: \n(generic) 1. A string as param definition. Gets appended to the command params signature.\n(implicit) 2. A mapping with the param name as the key and the type as the value. Custom types are accepted here. Same as the implicit v1 params syntax. \n(explicit) 3. A mapping with the following keys: `name` (required), `type` (required), `default` (None by default), `help` (Optional), `short` (Optional), `required` (False by default).", + description="The parameters of the command. Each entry takes one of three forms: " + "a shorthand mapping of the param name to its type, such as `name: str!` or `--count: int = 1`; " + "an explicit mapping with the keys `name` and `type` and optionally `default`, `help`, `short` and `required`; " + "or a string that is copied into the generated code as written " + "(a signature entry for Typer, a decorator for Click).", ) template: str = Field( default="", @@ -238,7 +263,7 @@ class CommandTemplate(BaseModel): params: list[ParamBlock] = Field( default=[], - description="A list of parameters for the command template. These parameters will be applied to any command that uses this template.", + description="A list of parameters for the command template. These parameters will be applied to any command that uses this template.", ) pre_run: PreRunBlock = Field( default=PreRunBlock(root=""), @@ -284,21 +309,22 @@ class CLIManifest(BaseModel): use_click: bool = Field(default=False, description="Generate CLI to Click instead of Typer") requires: list[str] = Field( default=[], - description="List of Python package dependencies for the CLI.Supports requirements specifier syntax.", + description="Python packages the CLI needs, as requirement specifiers such as `rich` or `tabulate>=0.9.0`. " + "Checked when the manifest is loaded and bundled by `cli build`.", ) includes: list[str] = Field( default=[], - description="List of external CLI manifests to include." - "Performs a deep merge of manifests sequentially in the order given to assemble a merged manifest. " - "and finally, deep merges the merged manifest with this manifest.", + description="Paths of other manifest files to merge into this one, relative to this file. " + "Keys set in this manifest win over included ones and lists are combined. " + "When two included files set the same key, the first one listed wins.", ) vars: dict[str, VarBlock] = Field( default_factory=dict, - description="Mapping defining manifest variables that can be referenced in any other blocks. " - "Environments variables can be used in this section with ${some_env_var} for dynamic parsing. " - "Supports jinja2 formatted expressions as values. " - "Interpolate defined vars in other blocks jinja2-styled {{ var_name }}.", + description="Variables for the manifest. Each one can be used anywhere in the file as `{{ name }}` " + "and is also defined as a module-level Python variable in the generated CLI. " + "A value is a string, a number, a boolean, or a Python expression written as `{expression}`. " + "String values are Jinja2 templates: they can use other vars, and `env` gives access to environment variables.", ) imports: str | list[str] = Field( default=[], @@ -315,7 +341,9 @@ class CLIManifest(BaseModel): default_factory=dict, description="A mapping containing any shared type definitions. " "These types can be referenced by name in the params section to provide type annotations " - "for args and options defined in the params section.", + "for args and options defined in the params section. " + "A value is a plain type such as `int`, optionally with a default (`int = 5`), or a complete definition: " + "`str = typer.Option(...)` for Typer, a decorator such as `@click.option(...)` for Click.", ) global_params: list[ParamBlock] = Field(default=[], description="Parameters applied to all commands") @@ -326,11 +354,8 @@ class CLIManifest(BaseModel): commands: dict[str, CommandBlock] | list[Command] = Field( default_factory=dict, - description="A mapping containing the command definitions for the CLI. " - "Each command should have a unique key- which can be either a group command or nested subcommands. " - "Nested subcommands are joined by '.' in between each level. " - "Aliases for commands can be separated in the key by '|'. " - "A special '(*)' wildcard can be used to spread the subcommand to all group-level commands", + description="The commands of the CLI, keyed by name. " + "In a key, `.` separates groups, `|` separates aliases and a leading `(*)` adds the command to every group.", ) callbacks: list[Callback] = Field( @@ -358,19 +383,39 @@ def is_safe_name(cls, value: str) -> str: ) return value - @field_validator("manifestVersion", mode="after") + @field_validator("vars", mode="after") @classmethod - def is_v3(cls, value: str) -> str: - if value.strip() == "v1": - raise ValueError( - "v1 schema is deprecated with cliffy >= 0.4.0. Please upgrade the manifest to the v3 schema." - ) - if value.strip() == "v2": + def var_names_are_identifiers(cls, value: dict[str, Any]) -> dict[str, Any]: + """A var becomes a module-level Python variable of the generated CLI.""" + for name, var in value.items(): + if isinstance(var, dict) and len(var) != 1: + raise ValueError( + f"vars.{name}: a Python expression is written as `{{expression}}`, one expression in braces. " + f"Found {len(var)}." + ) + if name.isidentifier() and not keyword.iskeyword(name) and name not in GENERATED_NAMES: + continue + + if name in GENERATED_NAMES: + problem = "is a name the generated CLI uses itself" + elif keyword.iskeyword(name): + problem = "is a Python keyword" + else: + problem = "is not a valid Python identifier" + suggestion = re.sub(r"\W", "_", name) + if not suggestion.isidentifier() or keyword.iskeyword(suggestion) or suggestion in GENERATED_NAMES: + suggestion = f"{suggestion}_" if suggestion.isidentifier() else f"_{suggestion}" raise ValueError( - "v2 schema is deprecated with cliffy >= 0.5.0. Please upgrade the manifest to the v3 schema." + f"vars.{name}: `{name}` {problem}. A var becomes a Python variable, so use letters, digits and `_` " + f"only, i.e. `{suggestion}`." ) - if value.strip() != "v3": - raise ValueError(f"Unrecognized manifest version {value}. Latest is {LATEST_SCHEMA_VERSION}.") + return value + + @field_validator("manifestVersion", mode="after") + @classmethod + def is_v3(cls, value: str) -> str: + if problem := manifest_version_problem(value): + raise ValueError(problem) return value @model_validator(mode="after") @@ -546,7 +591,8 @@ def get_raw_template(cls, cli_name: str, json_schema: bool) -> str: json_schema (bool): Flag to include JSON schema reference comment Returns: - str: A YAML-formatted template for a CLI manifest with default configuration + str: A YAML-formatted template for a CLI manifest with default configuration. It has one command + and one test, so that it runs and passes `cli test` as written. """ manifest = "" if json_schema: @@ -573,11 +619,13 @@ def get_raw_template(cls, cli_name: str, json_schema: bool) -> str: command_templates: {{}} -commands: {{}} +commands: + hello: print("Hello!") cli_options: {{}} -tests: [] +tests: + - hello: assert result_text == "Hello!" """ return manifest diff --git a/cliffy/parser.py b/cliffy/parser.py index 8d4b16c..088c1d1 100644 --- a/cliffy/parser.py +++ b/cliffy/parser.py @@ -159,9 +159,6 @@ def parse_param(self, param: CommandParam | GenericCommandParam | SimpleCommandP # a full typer definition, i.e. `name: str = typer.Option(..., prompt=True)`, is passed through as-is return f"{norm_param_name}: {param.raw_type.strip()}, " - if param_type in self.manifest.types: - return f"{norm_param_name}: {self.manifest.types[param_type]}," - aliases = [param.short] if param.short else None default_val = param.default @@ -170,6 +167,18 @@ def parse_param(self, param: CommandParam | GenericCommandParam | SimpleCommandP # a str default is data, not an expression: emit it as a string literal default_val = py_str(default_val) + if param_type in self.manifest.types: + definition = self.manifest.types[param_type].strip() + if "typer." in definition: + # a complete Typer definition replaces everything after the param name + return f"{norm_param_name}: {definition}," + # `N: int` or `N: int = 5` names a type (and a default): the param keeps its own name, kind and help + param_type, _, type_default = (part.strip() for part in definition.partition("=")) + if default_val is None and type_default: + default_val = type_default + elif isinstance(param, CommandParam) and param.default is not None and param_type == "str": + default_val = py_str(param.default) + return self.build_param_type( param_name=norm_param_name, param_type=param_type, diff --git a/cliffy/reloader.py b/cliffy/reloader.py index 931eab7..d1ee31b 100644 --- a/cliffy/reloader.py +++ b/cliffy/reloader.py @@ -2,16 +2,16 @@ import threading import time from datetime import datetime, timedelta -from pathlib import Path from watchdog.events import FileModifiedEvent, FileSystemEvent, FileSystemEventHandler from watchdog.observers import Observer +from watchdog.observers.api import BaseObserver from cliffy.builder import run_cli as cli_runner from cliffy.helper import out, out_err from cliffy.homer import save_metadata from cliffy.loader import Loader -from cliffy.transformer import Transformer +from cliffy.transformer import Transformer, source_files class Reloader(FileSystemEventHandler): @@ -22,12 +22,28 @@ def __init__(self, manifest_path: str, run_cli: bool, run_cli_args: tuple, bluep self.run_cli_args = run_cli_args self.blueprint = blueprint self.last_modified = datetime.now() + self.observer: BaseObserver | None = None + self.scheduled_directories: set[str] = set() + # the manifest, the manifests it includes and the Python file of its blueprint + self.watched_files: set[str] = set() + self.refresh_watched_files() super().__init__() + def refresh_watched_files(self) -> None: + """Reads which files the CLI is generated from. They change with the manifest, i.e. a new include.""" + self.watched_files = source_files(self.manifest_path, self.blueprint) + if self.observer: + for directory in sorted(self.watched_directories() - self.scheduled_directories): + self.observer.schedule(self, path=directory, recursive=False) + self.scheduled_directories.add(directory) + + def watched_directories(self) -> set[str]: + return {os.path.dirname(path) for path in self.watched_files} + def on_modified(self, event: FileSystemEvent) -> None: # compared as full paths: `endswith` also matched other-x.yaml when watching x.yaml - if event.is_directory or os.path.realpath(os.fsdecode(event.src_path)) != self.manifest_realpath: + if event.is_directory or os.path.realpath(os.fsdecode(event.src_path)) not in self.watched_files: return if not isinstance(event, FileModifiedEvent): @@ -37,6 +53,7 @@ def on_modified(self, event: FileSystemEvent) -> None: return self.last_modified = datetime.now() + self.refresh_watched_files() t = threading.Thread( target=self.reload, @@ -54,7 +71,8 @@ def on_modified(self, event: FileSystemEvent) -> None: def watch(cls, manifest_path: str, run_cli: bool, run_cli_args: tuple, blueprint: str | None = None) -> None: event_handler = cls(manifest_path, run_cli, run_cli_args, blueprint) observer = Observer() - observer.schedule(event_handler, path=str(Path(manifest_path).parent), recursive=False) + event_handler.observer = observer + event_handler.refresh_watched_files() observer.start() try: diff --git a/cliffy/rich.py b/cliffy/rich.py index cb97e1c..5d41f91 100644 --- a/cliffy/rich.py +++ b/cliffy/rich.py @@ -1,61 +1,83 @@ -## rich-click compat for rich-less support -from typing import Any +"""Click, Console and Table: from rich-click and rich when the `rich` extra is installed, plain otherwise. -__all__ = ["ClickGroup", "Console", "Table", "click"] +For the type checker `click` is always Click itself. rich-click has the same API, and whether it is +installed must not change what type-checks. +""" -try: - import rich_click as click # type: ignore - from rich.console import Console # type: ignore - from rich.table import Table # type: ignore - from rich_click import RichGroup # type: ignore +from typing import TYPE_CHECKING, Any - ClickGroup = RichGroup # type: ignore[no-redef] -except ImportError: - import click # type: ignore[no-redef] - from click import Group +__all__ = ["ClickGroup", "Console", "Table", "click", "print_rich_table"] + + +class PlainConsole: + """Stand-in for `rich.console.Console`""" + + def __init__(self) -> None: + pass + + def print(self, text: Any, **kwargs: Any) -> None: + if isinstance(text, PlainTable): + click.echo(text) + else: + print(text) + + +class PlainTable: + """Stand-in for `rich.table.Table`""" + + __slots__ = ("columns", "rows", "styles") + + def __init__(self) -> None: + self.columns: list[str] = [] + self.rows: list[list[str]] = [] + self.styles: list[str] = [] - ClickGroup: type[Group] = Group # type: ignore[no-redef] - - class Console: # type: ignore[no-redef] - def __init__(self) -> None: - pass - - def print(self, text: Any, **kwargs: Any) -> None: - if isinstance(text, Table): - click.echo(text) - else: - print(text) - - class Table: # type: ignore[no-redef] - __slots__ = ("columns", "rows", "styles") - - def __init__(self) -> None: - self.columns: list[str] = [] - self.rows: list[list[str]] = [] - self.styles: list[str] = [] - - def add_column(self, col: str, style: str = "") -> None: - self.columns.append(col) - self.styles.append(style) - - def add_row(self, *row: Any) -> None: - self.rows.append([*row]) - - def __str__(self) -> str: - text = ( - "".join( - [ - click.style(f"{col:10}", fg=style or None) - for col, style in zip(self.columns, self.styles, strict=True) - ] - ) - + "\n" + def add_column(self, col: str, style: str = "") -> None: + self.columns.append(col) + self.styles.append(style) + + def add_row(self, *row: Any) -> None: + self.rows.append([*row]) + + def __str__(self) -> str: + text = ( + "".join( + [ + click.style(f"{col:10}", fg=style or None) + for col, style in zip(self.columns, self.styles, strict=True) + ] ) - for row in self.rows: - for col in row: - text += f"{col:10}" - text += "\n" - return text + + "\n" + ) + for row in self.rows: + for col in row: + text += f"{col:10}" + text += "\n" + return text + + +if TYPE_CHECKING: + import click + from click import Group + + ClickGroup = Group + Console = PlainConsole + Table = PlainTable +else: + try: + import rich_click as click + from rich.console import Console + from rich.table import Table + from rich_click import RichGroup + + ClickGroup = RichGroup + except ImportError: + import click + from click import Group + + ClickGroup = Group + Console = PlainConsole + Table = PlainTable def print_rich_table(cols: list[str], rows: list[list[str]], styles: list[str]) -> None: diff --git a/cliffy/tester.py b/cliffy/tester.py index e6d681c..7b7d5ee 100644 --- a/cliffy/tester.py +++ b/cliffy/tester.py @@ -4,6 +4,7 @@ from typing import cast import pybash +from click import unstyle from click.testing import CliRunner as ClickCliRunner from click.testing import Result from pydantic import BaseModel, field_validator @@ -30,9 +31,9 @@ def is_shelled(cls, value: str) -> str: return value raise ValueError( - "Shell script for test case must start with > or $. " - "Use > to indicate a safe subprocess call. " - "Use $ to execute the command through the shell. WARNING: this can be dangerous." + "A shell step in tests must start with $ or >. " + "Use $ to run the command directly, without a shell. " + "Use > to run it through a shell (pipes into shell builtins, globs). WARNING: this can be dangerous." ) @@ -79,6 +80,14 @@ def invoke_shell(self, script: ShellScript) -> None: def invoke_test(self, command: str, script: str) -> Generator[Result, None, None]: result = self.runner.invoke(self.module.cli, command) + # Assertions compare text. rich styles help output when it thinks it runs on a CI system, so the + # same test would see `--name` on one machine and escape codes inside it on another. + for stream in ("stdout_bytes", "stderr_bytes", "output_bytes"): + captured = getattr(result, stream, None) + if isinstance(captured, bytes): + setattr( + result, stream, unstyle(captured.decode(self.runner.charset, "replace")).encode(self.runner.charset) + ) yield result code = compile(script, f"test_{self.T.cli.name}.py", "exec") exec(code, {}, {"result": result, "result_text": result.output.strip()}) diff --git a/cliffy/transformer.py b/cliffy/transformer.py index eca6eae..582e6c8 100644 --- a/cliffy/transformer.py +++ b/cliffy/transformer.py @@ -1,14 +1,18 @@ import os +import traceback +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO from typing import Any, Self, TextIO import yaml -from jinja2 import BaseLoader, Environment, FileSystemLoader +from jinja2 import BaseLoader, Environment, TemplateSyntaxError, meta from pydantic import ValidationError from cliffy.blueprint import ( BLUEPRINT_KEY, Blueprint, BlueprintError, + blueprint_file, build_manifest, normalize_reference, resolve_blueprint, @@ -17,9 +21,8 @@ from cliffy.commanders.click import ClickCommander from cliffy.commanders.typer import TyperCommander from cliffy.helper import check_requirement, exit_err, get_installed_package_versions, out -from cliffy.manifest import LATEST_SCHEMA_VERSION, CLIManifest, IncludeManifest +from cliffy.manifest import LATEST_SCHEMA_VERSION, CLIManifest, IncludeManifest, manifest_version_problem from cliffy.merger import cliffy_merger -from cliffy.parser import ScriptError class Transformer: @@ -52,7 +55,10 @@ def __init__( # manifests that are being resolved right now, outermost first: used to detect include cycles self.include_chain = (*include_chain, self.manifest_path) self.command_config = self.load_manifest(manifest_io) + # checked here, for every kind of file: an include and a blueprint spec have no model field for it self.manifest_version = self.command_config.pop("manifestVersion", LATEST_SCHEMA_VERSION) + if problem := manifest_version_problem(self.manifest_version): + exit_err(f"~ error validating {manifest_io.name}: {problem}") self.blueprint: Blueprint | None = None self.blueprint_ref = None if as_include else self.find_blueprint_reference(blueprint) if not self.blueprint_ref and self.command_config.get("includes"): @@ -73,7 +79,9 @@ def __init__( commander_cls = ClickCommander if self.manifest.use_click else TyperCommander try: self.cli = generate_cli(self.manifest, commander_cls=commander_cls) - except ScriptError as e: + except ValueError as e: + # a script that is not valid (ScriptError), or a name, param or template no code can be + # generated for: reported the same way by every command, without a traceback exit_err(f"~ error in {manifest_io.name}: {e}") if self.blueprint: self.cli.code = self.blueprint.post_codegen(self.cli.code) @@ -132,8 +140,7 @@ def resolve_include(self, path: str) -> Self: A relative path is looked up next to the including manifest first, then in the working directory. """ - candidates = [os.path.join(os.path.dirname(self.manifest_path), path), path] - include_path = next((candidate for candidate in candidates if os.path.isfile(candidate)), None) + include_path = self.find_include(self.manifest_path, path) if include_path is None: exit_err(f"~ error loading manifest: `{self.manifest_io.name}` includes `{path}`, which does not exist") @@ -144,12 +151,20 @@ def resolve_include(self, path: str) -> Self: with open(include_path, encoding="utf-8") as m: return type(self)(m, as_include=True, include_chain=self.include_chain) + @staticmethod + def find_include(manifest_path: str, path: str) -> str | None: + """Path of an included file: next to the including manifest first, then in the working directory.""" + candidates = [os.path.join(os.path.dirname(manifest_path), path), path] + return next((candidate for candidate in candidates if os.path.isfile(candidate)), None) + @staticmethod def load_manifest(manifest_io: TextIO) -> dict[str, Any]: + """Reads a manifest file: it is rendered as a Jinja template with its own `vars`, then parsed as YAML.""" try: manifest_path = os.path.realpath(manifest_io.name) with open(manifest_path, encoding="utf-8") as manifest_file: - raw_manifest = yaml.safe_load(manifest_file) + source = manifest_file.read() + raw_manifest = yaml.safe_load(source) if not isinstance(raw_manifest, dict): exit_err(f"~ error loading manifest: {manifest_io.name} must be a YAML mapping") @@ -157,20 +172,143 @@ def load_manifest(manifest_io: TextIO) -> dict[str, Any]: if not isinstance(manifest_vars, dict): exit_err(f"~ error loading manifest: vars in {manifest_io.name} must be a mapping") - # `env` is passed to the templates as-is. Environment values are data: they are never - # rendered as templates themselves. - render_context = {**manifest_vars, "env": os.environ} - var_env = Environment(loader=BaseLoader()) - interpolated_vars: dict[str, Any] = { - var_env.from_string(str(k)).render(render_context): var_env.from_string(str(v)).render(render_context) - for k, v in manifest_vars.items() - } - interpolated_vars["env"] = os.environ - manifest_env = Environment(loader=FileSystemLoader(manifest_path)).get_template("") - manifest = yaml.safe_load(manifest_env.render(interpolated_vars)) + context = resolve_vars(manifest_vars, manifest_io.name) + manifest = yaml.safe_load(render_manifest(source, context, manifest_io.name)) if not isinstance(manifest, dict): exit_err(f"~ error loading manifest: {manifest_io.name} must be a YAML mapping") return manifest except yaml.YAMLError as e: out(f"{e}") exit_err("~ error loading manifest") + + +JINJA = Environment(loader=BaseLoader()) +TEMPLATE_FILE = "