diff --git a/.custom_wordlist.txt b/.custom_wordlist.txt index bdb9b7c..05705e9 100644 --- a/.custom_wordlist.txt +++ b/.custom_wordlist.txt @@ -1,3 +1,4 @@ cna +buildd hda passt diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index a44f35d..491086e 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -20,9 +20,17 @@ jobs: node-version: "22" cache: "pnpm" + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # pin@v6.2.0 + with: + python-version: "3.13" + - name: Install dependencies run: | pnpm install + pip install git+https://github.com/gardenlinux/glrd.git@v4.1.0 + pip install git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.20 + pip install -r requirements.txt - name: Run aggregate run: | diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 7dc023b..3ff69d0 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -28,6 +28,8 @@ jobs: - name: Install dependencies run: | pnpm install + pip install git+https://github.com/gardenlinux/glrd.git@v4.1.0 + pip install git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.20 pip install -r requirements.txt - name: Run aggregate diff --git a/.github/workflows/woke.yml b/.github/workflows/woke.yml index e57805e..29f9206 100644 --- a/.github/workflows/woke.yml +++ b/.github/workflows/woke.yml @@ -20,10 +20,18 @@ jobs: node-version: "22" cache: "pnpm" + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # pin@v6.2.0 + with: + python-version: "3.13" + - name: Install dependencies run: | go install github.com/get-woke/woke@latest pnpm install + pip install git+https://github.com/gardenlinux/glrd.git@v4.1.0 + pip install git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.20 + pip install -r requirements.txt - name: Run aggregate run: | diff --git a/Makefile b/Makefile index 3b6bda3..74e4c4d 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ help: install: @echo "Installing dependencies..." pnpm install + pip install git+https://github.com/gardenlinux/glrd.git@v4.1.0 + pip install git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.20 + pip install -r requirements.txt dev: install pnpm run docs:dev diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index a90ef76..496e3cf 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -29,6 +29,17 @@ export default defineConfig({ /release-notes\/1877-12/, // Ignore LICENSE files (e.g. from aggregated repos) /LICENSE$/, + // Ignore broken links inside features for now + /features\/_ephemeral/, + /features\/_tpm2/, + /features\/bare/, + /features\/libc/, + /test\//, + /tests\//, + /samples\//, + /boot_modes/, + /cis\//, + /firewall\//, ], head: [ [ diff --git a/repos-config.json b/repos-config.json index 476f8bf..b175f34 100644 --- a/repos-config.json +++ b/repos-config.json @@ -6,10 +6,13 @@ "docs_path": "docs", "target_path": "projects/gardenlinux", "ref": "docs-ng", - "commit": "aafc4d98545ea7dcd458faff4e1b8d7e6fcb9af2", + "commit": "4c1d90299cef6cd9db3e800cdd9668949dc8ebec", "root_files": [ "CONTRIBUTING.md", - "SECURITY.md" + "SECURITY.md", + "features/*/README.md", + "features/*/info.yaml", + "flavors.yaml" ], "structure": { "tutorials": "tutorials", @@ -18,11 +21,7 @@ "reference": "reference", "contributing": "contributing" }, - "media_directories": [ - ".media", - "assets", - "_static" - ] + "media_directories": [".media", "assets", "_static"] }, { "name": "builder", @@ -31,11 +30,7 @@ "target_path": "projects/builder", "ref": "docs-ng", "commit": "086e74ff032c5f2a05989aef4f20ba69f94bdbf9", - "media_directories": [ - ".media", - "assets", - "_static" - ] + "media_directories": [".media", "assets", "_static"] }, { "name": "python-gardenlinux-lib", @@ -45,11 +40,7 @@ "ref": "docs-ng", "commit": "9142fccc3d83ab51759db7d328fa19166bc1df63", "structure": "sphinx", - "media_directories": [ - ".media", - "assets", - "_static" - ] + "media_directories": [".media", "assets", "_static"] }, { "name": "package-linux", @@ -58,11 +49,7 @@ "target_path": "projects/package-linux", "ref": "docs-ng", "commit": "841139441357a222ea10349f6d09a875ce9732d1", - "media_directories": [ - ".media", - "assets", - "_static" - ] + "media_directories": [".media", "assets", "_static"] }, { "name": "glrd", @@ -71,11 +58,7 @@ "target_path": "projects/glrd", "ref": "docs-ng", "commit": "c444721b728c0be0de2c19a0bc328928a0091d65", - "media_directories": [ - ".media", - "assets", - "_static" - ] + "media_directories": [".media", "assets", "_static"] } ] } diff --git a/repos-config.local.json b/repos-config.local.json index 55f8022..8fc6938 100644 --- a/repos-config.local.json +++ b/repos-config.local.json @@ -4,7 +4,13 @@ "name": "gardenlinux", "url": "file://../gardenlinux", "docs_path": "docs", - "root_files": ["CONTRIBUTING.md", "SECURITY.md"], + "root_files": [ + "CONTRIBUTING.md", + "SECURITY.md", + "features/*/README.md", + "features/*/info.yaml", + "flavors.yaml" + ], "target_path": "projects/gardenlinux", "structure": { "tutorials": "tutorials", diff --git a/requirements.txt b/requirements.txt index 90ebdb4..2db0426 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ codespell==2.4.2 pytest -glrd @ git+https://github.com/gardenlinux/glrd.git@main +pyyaml +# glrd @ git+https://github.com/gardenlinux/glrd.git@v4.2.0 +# gardenlinux @ git+https://github.com/gardenlinux/python-gardenlinux-lib.git@0.10.20 diff --git a/src/aggregate.py b/src/aggregate.py index 40cf389..967a6c7 100755 --- a/src/aggregate.py +++ b/src/aggregate.py @@ -21,6 +21,7 @@ ) from aggregation.releases import generate_release_docs from aggregation.release_notes import generate_release_notes_docs +from aggregation.flavor_matrix import generate_flavor_matrix_docs def transform_repo_docs( @@ -215,6 +216,15 @@ def main() -> int: save_config(str(config_path), repos) print(f"\n✓ Config updated: {config_path}") + + # Generate flavor matrix documentation after all repos are aggregated + # Use docs/projects/gardenlinux path since temp_dir is cleaned up + gardenlinux_docs_path = docs_dir / "projects" / "gardenlinux" + if gardenlinux_docs_path.exists(): + print(f"\n{'='*60}") + print("Generating flavor matrix documentation...") + print(f"{'='*60}\n") + generate_flavor_matrix_docs(docs_dir, gardenlinux_docs_path) # Generate release documentation from GLRD print(f"\n{'='*60}") diff --git a/src/aggregation/__init__.py b/src/aggregation/__init__.py index 48fc13c..5c4e302 100644 --- a/src/aggregation/__init__.py +++ b/src/aggregation/__init__.py @@ -17,6 +17,7 @@ process_all_markdown, ) from .releases import generate_release_docs +from .flavor_matrix import generate_flavor_matrix_docs __all__ = [ # Models @@ -38,4 +39,6 @@ "process_all_markdown", # Releases "generate_release_docs", + # Flavor Matrix + "generate_flavor_matrix_docs", ] \ No newline at end of file diff --git a/src/aggregation/fetcher.py b/src/aggregation/fetcher.py index 02dadcd..1f033e4 100644 --- a/src/aggregation/fetcher.py +++ b/src/aggregation/fetcher.py @@ -10,6 +10,23 @@ from .models import RepoConfig, AggregateResult +def _convert_to_git_pattern(pattern: str) -> str: + """ + Convert Python glob pattern to git sparse-checkout compatible pattern. + + Git doesn't support **, so convert to folder prefix. + e.g., "features/**/*.md" -> "features/*" + """ + if "**" in pattern: + parts = pattern.split("/") + for i, part in enumerate(parts): + if "**" in part: + parts[i] = "*" + return "/".join(parts[:i+1]) + return pattern + return pattern + + class DocsFetcher: """Handles fetching documentation from remote or local repositories.""" @@ -73,11 +90,13 @@ def _fetch_remote( ) # Configure sparse checkout patterns + # Git sparse-checkout doesn't support **, so convert to folder prefixes sparse_checkout_file = temp_dir / ".git" / "info" / "sparse-checkout" with open(sparse_checkout_file, "w") as f: f.write(f"{repo.docs_path}/*\n") for root_file in repo.root_files: - f.write(f"{root_file}\n") + converted = _convert_to_git_pattern(root_file) + f.write(f"{converted}\n") # Fetch and checkout print(" Cloning (sparse checkout)...") @@ -215,11 +234,14 @@ def _copy_docs(source: Path, dest: Path) -> None: @staticmethod def _copy_root_files(repo_root: Path, root_files: list, dest: Path) -> None: """ - Copy specified root-level files from repository. + Copy specified root-level files and directories from repository. + + Supports glob patterns like "features/*/*.md" to match specific files + without copying entire directories. Args: repo_root: Root directory of the repository - root_files: List of filenames to copy + root_files: List of filenames/directories/patterns to copy dest: Destination directory """ if not root_files: @@ -227,10 +249,41 @@ def _copy_root_files(repo_root: Path, root_files: list, dest: Path) -> None: print(" Copying root files") for filename in root_files: - src = repo_root / filename - if src.exists(): - target = dest / src.name - shutil.copy2(src, target) - print(f" ✓ {filename}") + clean_name = filename.rstrip("/") + + if '*' in clean_name or '?' in clean_name or '[' in clean_name: + matches = list(repo_root.glob(clean_name)) + if not matches: + print(f" Warning: {filename} not found (no matches)") + continue + + for src in matches: + rel_path = src.relative_to(repo_root) + target = dest / rel_path + if src.is_dir(): + try: + shutil.copytree(src, target, dirs_exist_ok=True, symlinks=False) + print(f" ✓ {filename} -> {rel_path} (directory)") + except Exception as e: + print(f" Warning: Failed to copy {rel_path}: {e}") + else: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, target) + print(f" ✓ {filename} -> {rel_path}") else: - print(f" Warning: {filename} not found") + src = repo_root / clean_name + if src.exists(): + target = dest / src.name + if src.is_dir(): + try: + shutil.copytree(src, target, dirs_exist_ok=True, symlinks=False) + print(f" ✓ {filename} (directory)") + except Exception as e: + print(f" Warning: Failed to copy {filename}: {e}") + else: + shutil.copy2(src, target) + print(f" ✓ {filename}") + else: + print(f" Warning: {filename} not found") + + diff --git a/src/aggregation/flavor_matrix.py b/src/aggregation/flavor_matrix.py new file mode 100644 index 0000000..45d678a --- /dev/null +++ b/src/aggregation/flavor_matrix.py @@ -0,0 +1,140 @@ +"""Generate flavor matrix documentation from flavors.yaml and feature dependencies.""" + +import re +import yaml +from pathlib import Path +from typing import Optional, List, Tuple + +from gardenlinux.features import Parser as FeaturesParser +from gardenlinux.flavors.parser import Parser as FlavorsParser + + +def get_flavor_list(gardenlinux_repo_dir: Path) -> Optional[dict]: + """Get flavor list by parsing flavors.yaml directly.""" + flavors_file = gardenlinux_repo_dir / "flavors.yaml" + + if not flavors_file.exists(): + print(f"Warning: flavors.yaml not found at {flavors_file}") + return None + + try: + # Read and parse the YAML file + flavors_data = yaml.safe_load(flavors_file.read_text()) + + # Use the FlavorsParser to get combinations + flavors_parser = FlavorsParser(flavors_data) + combinations = flavors_parser.filter() + + # Group by architecture + by_arch = {} + for arch, combination in combinations: + if arch not in by_arch: + by_arch[arch] = [] + by_arch[arch].append(combination) + + return by_arch + except Exception as e: + print(f"Error parsing flavors.yaml: {e}") + return None + + +def generate_flavor_matrix_docs( + docs_dir: Path, gardenlinux_repo_dir: Path +) -> bool: + """ + Generate flavor matrix page from flavors.yaml and feature dependencies. + + Args: + docs_dir: The docs directory to write output to + gardenlinux_repo_dir: Path to the fetched gardenlinux repository + + Returns: + True if successful, False otherwise + """ + print("\nGenerating flavor matrix documentation...") + + # Step 1: Get flavor list from flavors.yaml + flavors_by_arch = get_flavor_list(gardenlinux_repo_dir) + if flavors_by_arch is None: + print("Warning: Could not parse flavors - skipping flavor matrix generation") + return False + + # Step 2: Build features parser for recursive resolution + features_dir = gardenlinux_repo_dir + if not features_dir.exists(): + print(f"Warning: features directory not found at {features_dir}") + return False + + try: + features_parser = FeaturesParser(str(features_dir)) # Default feature_dir_name is "features" + except Exception as e: + print(f"Failed to initialize features parser: {e}") + return False + + # Step 3: Generate table rows + rows = [] + for arch, flavors in flavors_by_arch.items(): + for flavor in flavors: + # Strip the arch suffix (e.g., "aws-gardener_prod-amd64" -> "aws-gardener_prod") + flavor_base = flavor.replace(f"-{arch}", "") + + # Explicit features from cname + explicit = FeaturesParser.get_flavor_as_feature_set(flavor_base) + + # Full recursive feature set + try: + all_features = features_parser.filter_as_list(flavor_base) + except Exception as e: + print(f"Warning: Failed to resolve features for {flavor_base}: {e}") + all_features = explicit[:] + + # Recursive-only = all minus explicit + recursive_only = [f for f in all_features if f not in explicit] + + rows.append( + { + "flavor": flavor_base, + "arch": arch, + "explicit": explicit, + "recursive": recursive_only, + } + ) + + # Step 4: Render markdown with hyperlinks + def link(feature: str) -> str: + return f"[{feature}](features/{feature}.md)" + + header = "| Flavor | Arch | Features | Recursive Features |\n" + header += "|--------|------|----------|--------------------|" + + table_rows = [] + for row in sorted(rows, key=lambda r: (r["flavor"], r["arch"])): + explicit_links = ", ".join(link(f) for f in row["explicit"]) + recursive_links = ", ".join(link(f) for f in row["recursive"]) + table_rows.append( + f"| {row['flavor']} | {row['arch']} | {explicit_links} | {recursive_links} |" + ) + + table = header + "\n" + "\n".join(table_rows) + + # Step 5: Append table to existing aggregated file (keeps frontmatter and content) + output_file = docs_dir / "reference" / "flavor-matrix.md" + if output_file.exists(): + try: + existing_content = output_file.read_text() + # Append table + content = existing_content + f""" + +{table} +""" + except Exception as e: + print(f"Warning: Could not read existing file: {e}") + + output_dir = docs_dir / "reference" + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "flavor-matrix.md" + output_file.write_text(content) + print(f" Created: {output_file}") + + print("Flavor matrix generation complete.") + return True