diff --git a/.gitignore b/.gitignore index 17be597..e2c9e60 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ venv/ build/ *.egg-info/ uv.lock +.pytest_cache +.ruff_cache # Dev files @@ -17,4 +19,5 @@ test.py test_async.py canvas_client.py .env.local +excluded_specs.toml testing/ \ No newline at end of file diff --git a/README.md b/README.md index 4bf494b..8af14d2 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,85 @@ canvas_api_builder build-all-apis --specs-folder specs/ --output-folder apis/ For more information on using this command run `canvas_api_builder build-all-apis --help` +### Excluding specs from processing + +Some Canvas spec files contain malformed parameters that cause code generation to fail or produce invalid Python. You can maintain a local TOML file to exclude these specs from `build-all-apis`, `rebuild-apis`, and `update-spec-files`. + +Copy the provided example file to get started: + +```bash +cp excluded_specs.toml.example excluded_specs.toml +``` + +The file format is a single list of spec filenames: + +```toml +# excluded_specs.toml +excluded = [ + "quiz_extensions.json", + "plagiarism_detection.json", +] +``` + +Pass it to any command that processes multiple specs via `--exclude-file`: + +```bash +canvas_api_builder update-spec-files \ + --specs-folder specs/ \ + --exclude-file excluded_specs.toml + +canvas_api_builder build-all-apis \ + --specs-folder specs/ \ + --output-folder apis/ \ + --exclude-file excluded_specs.toml + +canvas_api_builder rebuild-apis \ + --specs-folder specs/ \ + --apifolder-path apis/ \ + --exclude-file excluded_specs.toml +``` + +>`excluded_specs.toml` is gitignored by default — each user maintains their own list locally. + +## Generating LLM documentation + +The `canopy_docs` script generates LLM-readable reference files from your project's generated API modules. This is useful for providing context to LLMs when building applications on top of Canopy. + +### Generate the Canopy framework reference + +```bash +canopy_docs generate-llms +``` + +Writes a static `llms.txt` describing `CanvasSession`, `CanvasClient`, `CanvasAPIError`, pagination behaviour, async usage, and common usage patterns. + +### Generate an index of all API methods + +```bash +canopy_docs generate-index --apis-folder apis/ +``` + +Parses your generated API files using AST and emits a compact method index with signatures and inferred return types, grouped by class. + +### Generate both in one step + +```bash +canopy_docs generate-all --apis-folder apis/ +``` + +### Excluding specs from the index + +The same `excluded_specs.toml` file used with `canvas_api_builder` can be passed to `generate-index` and `generate-all` to keep excluded specs out of the generated documentation: + +```bash +canopy_docs generate-index \ + --apis-folder apis/ \ + --exclude-file excluded_specs.toml + +canopy_docs generate-all \ + --apis-folder apis/ \ + --exclude-file excluded_specs.toml +``` ## Usage in your project @@ -199,7 +278,7 @@ async def main(): tasks = [get_user_details_async(student_id) for student_id in student_ids] for coro in asyncio.as_completed(tasks): name = await coro - print(name["name"]) + print(name) print( f"Total time (asynchronous print as completed): {perf_counter() - time_before}" ) diff --git a/canopy/scripts/canopy_docs.py b/canopy/scripts/canopy_docs.py index e8b9190..06f7f72 100644 --- a/canopy/scripts/canopy_docs.py +++ b/canopy/scripts/canopy_docs.py @@ -7,6 +7,7 @@ """ import ast +import tomllib from pathlib import Path import click @@ -283,6 +284,29 @@ async def main(): """ +# ── Excluded specs loader ──────────────────────────────────────────── + + +def load_excluded_specs(exclude_file: Path | None) -> set[str]: + """Load a set of excluded spec filenames from a TOML file. + + The TOML file must contain a top-level ``excluded`` key whose value is a + list of spec filenames (e.g. ``["quiz_extensions.json"]``). Returns an + empty set if *exclude_file* is None. + """ + if exclude_file is None: + return set() + with exclude_file.open("rb") as f: + data = tomllib.load(f) + excluded = data.get("excluded", []) + if not isinstance(excluded, list): + raise click.BadParameter( + f"'excluded' in {exclude_file} must be a list of filenames, " + f"got {type(excluded).__name__}." + ) + return set(excluded) + + # ── AST helpers ───────────────────────────────────────────────────── @@ -363,13 +387,19 @@ def _collect_api_files( apis_folder: Path, sync_only: bool, async_only: bool, + excluded_specs: set[str], ) -> list[Path]: - excluded = {"canvas_client.py", "__init__.py"} - files = sorted(p for p in apis_folder.iterdir() if p.suffix == ".py" and p.name not in excluded) + excluded_py_files = {"canvas_client.py", "__init__.py"} + files = sorted( + p for p in apis_folder.iterdir() if p.suffix == ".py" and p.name not in excluded_py_files + ) if sync_only: - return [p for p in files if "_async" not in p.stem] + files = [p for p in files if "_async" not in p.stem] if async_only: - return [p for p in files if "_async" in p.stem] + files = [p for p in files if "_async" in p.stem] + if excluded_specs: + # Match against the source spec name (strip _async suffix, swap .py → .json) + files = [p for p in files if f"{p.stem.replace('_async', '')}.json" not in excluded_specs] return files @@ -399,11 +429,19 @@ def cli() -> None: ) @click.option("--sync-only", is_flag=True, default=False, help="Only index sync modules.") @click.option("--async-only", is_flag=True, default=False, help="Only index async modules.") +@click.option( + "-e", + "--exclude-file", + default=None, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="TOML file listing spec filenames to exclude from the index.", +) def generate_index( apis_folder: Path, output_file: Path, sync_only: bool, async_only: bool, + exclude_file: Path | None, ) -> None: """Generate a compact LLM-readable index of all generated API methods. @@ -412,7 +450,11 @@ def generate_index( class. Canopy kwargs (as_user_id, do_not_process, no_data) are collapsed to '...' since they are identical on every method and documented in llms.txt. """ - api_files = _collect_api_files(apis_folder, sync_only, async_only) + excluded_specs = load_excluded_specs(exclude_file) + if excluded_specs: + click.echo(f"Excluding {len(excluded_specs)} spec(s): {', '.join(sorted(excluded_specs))}") + + api_files = _collect_api_files(apis_folder, sync_only, async_only, excluded_specs) blocks: list[str] = [ "# Canopy API Index", @@ -473,11 +515,19 @@ def generate_llms(output_file: Path) -> None: type=click.Path(file_okay=False, writable=True, path_type=Path), help="Folder to write llms.txt and apis_index.txt into.", ) +@click.option( + "-e", + "--exclude-file", + default=None, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="TOML file listing spec filenames to exclude from the index.", +) @click.pass_context def generate_all( ctx: click.Context, apis_folder: Path, output_folder: Path, + exclude_file: Path | None, ) -> None: """Run generate-llms and generate-index in one step. @@ -492,6 +542,7 @@ def generate_all( output_file=output_folder / "apis_index.txt", sync_only=False, async_only=False, + exclude_file=exclude_file, ) diff --git a/canopy/scripts/canvas_api_builder.py b/canopy/scripts/canvas_api_builder.py index 3587197..5c12722 100644 --- a/canopy/scripts/canvas_api_builder.py +++ b/canopy/scripts/canvas_api_builder.py @@ -2,6 +2,7 @@ import keyword import random import time +import tomllib from operator import itemgetter from pathlib import Path from typing import IO @@ -10,7 +11,25 @@ import httpx from jinja2 import Environment, FileSystemLoader, PackageLoader -blacklist: list[str] = [] + +def load_excluded_specs(exclude_file: Path | None) -> set[str]: + """Load a set of excluded spec filenames from a TOML file. + + The TOML file must contain a top-level ``excluded`` key whose value is a + list of spec filenames (e.g. ``["quiz_extensions.json"]``). Returns an + empty set if *exclude_file* is None. + """ + if exclude_file is None: + return set() + with exclude_file.open("rb") as f: + data = tomllib.load(f) + excluded = data.get("excluded", []) + if not isinstance(excluded, list): + raise click.BadParameter( + f"'excluded' in {exclude_file} must be a list of filenames, " + f"got {type(excluded).__name__}." + ) + return set(excluded) def fix_param_name(name: str) -> str: @@ -166,33 +185,47 @@ def build_canvas_client_file(apis_folder: Path) -> None: help="Path to output the API file to.", ) @click.option("--generate-async", is_flag=True, default=False, help="Generate async version") +@click.option( + "-e", + "--exclude-file", + default=None, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="TOML file listing spec filenames to exclude from processing.", +) @click.pass_context def build_all_apis( ctx: click.Context, specs_folder: Path, output_folder: Path, generate_async: bool, + exclude_file: Path | None, ) -> None: """Build all APIs from downloaded specfiles.""" + excluded = load_excluded_specs(exclude_file) + if excluded: + click.echo(f"Excluding {len(excluded)} spec(s): {', '.join(sorted(excluded))}") + for spec_path in specs_folder.iterdir(): - if spec_path.name not in blacklist: - if not generate_async: - with spec_path.open() as f: - ctx.invoke( - build_api_from_specfile, - specfile=f, - api_name=None, - output_folder=output_folder, - ) - else: - with spec_path.open() as f: - ctx.invoke( - build_api_from_specfile, - specfile=f, - api_name=None, - output_folder=output_folder, - generate_async=True, - ) + if spec_path.name in excluded: + click.echo(f"Skipping excluded spec: {spec_path.name}") + continue + if not generate_async: + with spec_path.open() as f: + ctx.invoke( + build_api_from_specfile, + specfile=f, + api_name=None, + output_folder=output_folder, + ) + else: + with spec_path.open() as f: + ctx.invoke( + build_api_from_specfile, + specfile=f, + api_name=None, + output_folder=output_folder, + generate_async=True, + ) # Rebuild APIs @@ -211,16 +244,36 @@ def build_all_apis( type=click.Path(file_okay=False, writable=True, path_type=Path), help="Path for API files", ) +@click.option( + "-e", + "--exclude-file", + default=None, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="TOML file listing spec filenames to exclude from processing.", +) @click.pass_context -def rebuild_apis(ctx: click.Context, specs_folder: Path, apifolder_path: Path) -> None: +def rebuild_apis( + ctx: click.Context, + specs_folder: Path, + apifolder_path: Path, + exclude_file: Path | None, +) -> None: """Rebuild all APIs from downloaded specfiles.""" excluded_files = {"canvas_client.py", "__init__.py"} + excluded_specs = load_excluded_specs(exclude_file) + if excluded_specs: + click.echo(f"Excluding {len(excluded_specs)} spec(s): {', '.join(sorted(excluded_specs))}") + for api_path in apifolder_path.iterdir(): if not api_path.is_file() or api_path.name in excluded_files: continue is_async = "async" in api_path.stem base_stem = api_path.stem.replace("_async", "") if is_async else api_path.stem - spec_path = specs_folder / f"{base_stem}.json" + spec_filename = f"{base_stem}.json" + if spec_filename in excluded_specs: + click.echo(f"Skipping excluded spec: {spec_filename}") + continue + spec_path = specs_folder / spec_filename with spec_path.open() as f: ctx.invoke( build_api_from_specfile, @@ -246,9 +299,17 @@ def rebuild_apis(ctx: click.Context, specs_folder: Path, apifolder_path: Path) - default=None, help="Download a single spec file by name (e.g. assignments.json)", ) -def update_spec_files(specs_folder: Path, spec_name: str | None) -> None: +@click.option( + "-e", + "--exclude-file", + default=None, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="TOML file listing spec filenames to exclude from downloading.", +) +def update_spec_files(specs_folder: Path, spec_name: str | None, exclude_file: Path | None) -> None: """Update spec files from Instructure API docs.""" base_url = "https://canvas.instructure.com/doc/api/" + excluded = load_excluded_specs(exclude_file) if spec_name: spec_names = [spec_name] @@ -257,6 +318,11 @@ def update_spec_files(specs_folder: Path, spec_name: str | None) -> None: spec["path"][1:] for spec in httpx.get(f"{base_url}api-docs.json").json()["apis"] ] + if excluded: + before = len(spec_names) + spec_names = [n for n in spec_names if n not in excluded] + click.echo(f"Excluding {before - len(spec_names)} spec(s) from download.") + total = len(spec_names) for i, name in enumerate(spec_names, 1): click.echo(f"Fetching {name} ({i}/{total})...") diff --git a/excluded_specs.toml.example b/excluded_specs.toml.example new file mode 100644 index 0000000..7072078 --- /dev/null +++ b/excluded_specs.toml.example @@ -0,0 +1,18 @@ +# excluded_specs.toml +# +# List spec filenames to skip during build and download commands. +# Copy this file to excluded_specs.toml and pass it via --exclude-file: +# +# canvas_api_builder build-all-apis \ +# --specs-folder specs/ \ +# --output-folder apis/ \ +# --exclude-file excluded_specs.toml +# +# Specs are often excluded because they contain malformed parameters that +# cause code generation to fail or produce invalid Python. + +excluded = [ + # "quiz_extensions.json", # malformed parameter names + # "plagiarism_detection.json", + # "late_policy.json", +]