From 58bd5a5689b10d80642b6340707bbc747a585c69 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:26:59 +0100 Subject: [PATCH 1/3] feat(registry): add --only-entities flag to registry build Rigenera solo la sezione entities del registry.json dopo un cambio in semantic_types.yaml. Evita di rigenerare datasets, marts, signals. Uso: toolkit registry build --repo --only-entities --write --- toolkit/cli/cmd_registry.py | 61 +++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/toolkit/cli/cmd_registry.py b/toolkit/cli/cmd_registry.py index bf336a2..a512a23 100644 --- a/toolkit/cli/cmd_registry.py +++ b/toolkit/cli/cmd_registry.py @@ -101,6 +101,11 @@ def registry_build( flat: bool = typer.Option(False, "--flat", help="Layout flat per clean+mart (no year)"), write: bool = typer.Option(False, "--write", help="Scrive registry.json (default: dry-run)"), out: str = typer.Option("registry", "--out", help="Dir di output (default: registry)"), + only_entities: bool = typer.Option( + False, + "--only-entities", + help="Rigenera solo la sezione entities (dopo cambio semantic_types)", + ), ) -> None: """Genera registry.json del repo (auto-discovery, fusion ADR). @@ -112,12 +117,64 @@ def registry_build( toolkit registry build # dry-run sul repo corrente toolkit registry build --write # scrive registry/registry.json toolkit registry build --prefix eurostat --flat --write + toolkit registry build --write --only-entities # solo grafo dopo cambio semantic_types """ - from toolkit.registry.builders import build_registry + from toolkit.registry.builders import build_registry, build_entity_graph from toolkit.registry.layout import RepoLayout, repo_dataset_dirs from toolkit.registry.paths import PathContract repo_root = Path(repo).resolve() if repo else Path.cwd() + out_dir = repo_root / out + existing_path = out_dir / "registry.json" + + # ── Modalità --only-entities: rigenera solo la sezione entities ── + if only_entities: + if not existing_path.is_file(): + typer.echo( + "ERRORE: registry.json non esistente — usa `registry build --write` prima", err=True + ) + raise typer.Exit(code=1) + + existing = json.loads(existing_path.read_text(encoding="utf-8")) + + # Ricostruisci il catalogo per avere le colonne con semantic_type + sections = repo_dataset_dirs(repo_root) + layout = RepoLayout( + repo_root=repo_root, + dataset_dirs=sections, + source_repo=_git_source_repo(repo_root), + ) + contract = PathContract(prefix=prefix, clean_layout="flat" if flat else "year") + from toolkit.registry.builders import build_clean_catalog + + catalog, _ = build_clean_catalog(layout, path_contract=contract) + + # Rigenera solo il grafo + graph = build_entity_graph(catalog) + + # Aggiorna la sezione entities nel registry esistente + existing["entities"] = { + "entities": graph.get("entities", {}), + "bridges": graph.get("bridges", []), + } + + typer.echo( + f"entities aggiornate: {len(existing['entities']['entities'])} entità, " + f"{len(existing['entities']['bridges'])} bridge" + ) + + if not write: + typer.echo("Dry-run: usa --write per scrivere il file.") + return + + existing_path.write_text( + json.dumps(existing, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + typer.echo(f"scritto {existing_path} (solo entities)") + return + + # ── Modalità normale: rigenera tutto ── sections = repo_dataset_dirs(repo_root) if not sections: typer.echo( @@ -137,10 +194,8 @@ def registry_build( mart_layout="flat" if flat else "year", ) - out_dir = repo_root / out existing_catalog = None existing_signals = None - existing_path = out_dir / "registry.json" if existing_path.is_file(): try: existing = json.loads(existing_path.read_text(encoding="utf-8")) From 2f3ac6039559a473fe60f196cada18f681a217ec Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:35:03 +0100 Subject: [PATCH 2/3] test(registry): add --only-entities tests (dry-run, preservazione, errore) 3 test: - test_only_entities_dry_run: dry-run restituisce entities senza scrivere - test_only_entities_preserves_other_sections: write preserva datasets/marts/signals - test_only_entities_fails_without_existing_registry: errore se registry.json mancante --- tests/test_cli_registry.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/test_cli_registry.py b/tests/test_cli_registry.py index d225dfb..7861f49 100644 --- a/tests/test_cli_registry.py +++ b/tests/test_cli_registry.py @@ -92,3 +92,84 @@ def test_build_no_datasets_dir_fails(tmp_path: Path) -> None: result = runner.invoke(app, ["registry", "build", "--repo", str(tmp_path)], obj={}) assert result.exit_code != 0 assert "nessuna sezione dati" in result.stderr + + +# --------------------------------------------------------------------------- +# --only-entities flag +# --------------------------------------------------------------------------- + + +@pytest.mark.pure_unit +def test_only_entities_dry_run(tmp_path: Path) -> None: + """--only-entities dry-run: ricalcola solo le entities senza scrivere.""" + _make_repo(tmp_path, with_run=False) + out = tmp_path / "registry" + # Prima genera il registry completo + result = runner.invoke( + app, ["registry", "build", "--repo", str(tmp_path), "--write", "--out", str(out)], obj={} + ) + assert result.exit_code == 0, result.stdout + result.stderr + + # Poi dry-run --only-entities + result2 = runner.invoke( + app, + ["registry", "build", "--repo", str(tmp_path), "--only-entities", "--out", str(out)], + obj={}, + ) + assert result2.exit_code == 0, result2.stdout + result2.stderr + assert "entities" in result2.stdout + assert "Dry-run" in result2.stdout + + +@pytest.mark.pure_unit +def test_only_entities_preserves_other_sections(tmp_path: Path) -> None: + """--only-entities --write: rigenera solo entities, preserva datasets/marts/signals.""" + _make_repo(tmp_path, with_run=False) + out = tmp_path / "registry" + # Prima genera tutto + runner.invoke( + app, ["registry", "build", "--repo", str(tmp_path), "--write", "--out", str(out)], obj={} + ) + payload_before = json.loads((out / "registry.json").read_text(encoding="utf-8")) + ds_before = len(payload_before["datasets"]) + marts_before = len(payload_before["marts"]) + signals_before = len(payload_before["signals"]) + + # Poi --only-entities --write + result = runner.invoke( + app, + [ + "registry", + "build", + "--repo", + str(tmp_path), + "--only-entities", + "--write", + "--out", + str(out), + ], + obj={}, + ) + assert result.exit_code == 0, result.stdout + result.stderr + + payload_after = json.loads((out / "registry.json").read_text(encoding="utf-8")) + # Datasets, marts, signals invariati + assert len(payload_after["datasets"]) == ds_before + assert len(payload_after["marts"]) == marts_before + assert len(payload_after["signals"]) == signals_before + # Entities potrebbe essere diverso (ricostruito) + assert "entities" in payload_after + + +@pytest.mark.pure_unit +def test_only_entities_fails_without_existing_registry(tmp_path: Path) -> None: + """--only-entities senza registry.json esistente → errore.""" + _make_repo(tmp_path, with_run=False) + out = tmp_path / "registry" + result = runner.invoke( + app, + ["registry", "build", "--repo", str(tmp_path), "--only-entities", "--out", str(out)], + obj={}, + ) + assert result.exit_code != 0 + assert "registry.json non esistente" in result.stderr From 128e3decb9fdac201fc9bec8e143612520a0a06b Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:38:10 +0100 Subject: [PATCH 3/3] refactor(registry): extract shared setup into _setup_build Elimina la duplicazione tra path normale e --only-entities. _setup_build() raggruppa: repo_dataset_dirs, RepoLayout, PathContract. --- toolkit/cli/cmd_registry.py | 71 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/toolkit/cli/cmd_registry.py b/toolkit/cli/cmd_registry.py index a512a23..bf41bf6 100644 --- a/toolkit/cli/cmd_registry.py +++ b/toolkit/cli/cmd_registry.py @@ -95,6 +95,38 @@ def _git_source_repo(repo_root: Path) -> str: return repo_root.name +def _setup_build( + repo_root: Path, + prefix: str, + flat: bool, + out_dir: Path, +) -> tuple: + """Setup condiviso per registry build: layout + contract + existing.""" + from toolkit.registry.layout import RepoLayout, repo_dataset_dirs + from toolkit.registry.paths import PathContract + + sections = repo_dataset_dirs(repo_root) + if not sections: + typer.echo( + f"ERRORE: nessuna sezione dati in {repo_root} (nessuna dir con {{slug}}/dataset.yml)", + err=True, + ) + raise typer.Exit(code=1) + + layout = RepoLayout( + repo_root=repo_root, + dataset_dirs=sections, + source_repo=_git_source_repo(repo_root), + ) + contract = PathContract( + prefix=prefix, + clean_layout="flat" if flat else "year", + mart_layout="flat" if flat else "year", + ) + existing_path = out_dir / "registry.json" + return layout, contract, existing_path + + def registry_build( repo: str = typer.Option(None, "--repo", help="Root del repo (default: CWD)"), prefix: str = typer.Option("", "--prefix", help="Prefisso GCS (es. 'eurostat')"), @@ -119,13 +151,11 @@ def registry_build( toolkit registry build --prefix eurostat --flat --write toolkit registry build --write --only-entities # solo grafo dopo cambio semantic_types """ - from toolkit.registry.builders import build_registry, build_entity_graph - from toolkit.registry.layout import RepoLayout, repo_dataset_dirs - from toolkit.registry.paths import PathContract + from toolkit.registry.builders import build_registry, build_entity_graph, build_clean_catalog repo_root = Path(repo).resolve() if repo else Path.cwd() out_dir = repo_root / out - existing_path = out_dir / "registry.json" + layout, contract, existing_path = _setup_build(repo_root, prefix, flat, out_dir) # ── Modalità --only-entities: rigenera solo la sezione entities ── if only_entities: @@ -136,23 +166,9 @@ def registry_build( raise typer.Exit(code=1) existing = json.loads(existing_path.read_text(encoding="utf-8")) - - # Ricostruisci il catalogo per avere le colonne con semantic_type - sections = repo_dataset_dirs(repo_root) - layout = RepoLayout( - repo_root=repo_root, - dataset_dirs=sections, - source_repo=_git_source_repo(repo_root), - ) - contract = PathContract(prefix=prefix, clean_layout="flat" if flat else "year") - from toolkit.registry.builders import build_clean_catalog - catalog, _ = build_clean_catalog(layout, path_contract=contract) - - # Rigenera solo il grafo graph = build_entity_graph(catalog) - # Aggiorna la sezione entities nel registry esistente existing["entities"] = { "entities": graph.get("entities", {}), "bridges": graph.get("bridges", []), @@ -175,25 +191,6 @@ def registry_build( return # ── Modalità normale: rigenera tutto ── - sections = repo_dataset_dirs(repo_root) - if not sections: - typer.echo( - f"ERRORE: nessuna sezione dati in {repo_root} (nessuna dir con {{slug}}/dataset.yml)", - err=True, - ) - raise typer.Exit(code=1) - - layout = RepoLayout( - repo_root=repo_root, - dataset_dirs=sections, - source_repo=_git_source_repo(repo_root), - ) - contract = PathContract( - prefix=prefix, - clean_layout="flat" if flat else "year", - mart_layout="flat" if flat else "year", - ) - existing_catalog = None existing_signals = None if existing_path.is_file():