diff --git a/Justfile b/Justfile index be484f8..59eb317 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,10 @@ set windows-shell := ["pwsh", "-c"] set dotenv-load := true export PYTHONPATH := "src" +export PYTHONUTF8 := "1" +export MLFLOW_TRACKING_URI := env_var("MLFLOW_TRACKING_URI") +export MLFLOW_TRACKING_USERNAME := env_var("MLFLOW_TRACKING_USERNAME") +export MLFLOW_TRACKING_PASSWORD := env_var("MLFLOW_TRACKING_PASSWORD") # Ensures Docker Desktop is running before any build step. # Attempts auto-launch on Windows; waits up to 90s for daemon to become ready. @@ -40,7 +44,7 @@ train-prod: export-model: uv run python -m scripts.export_model -build-api: check-docker export-model +build-api: check-docker docker build -f Dockerfile.api -t rossmann-api:latest . build-ui: check-docker diff --git a/scripts/clean_artifacts.py b/scripts/clean_artifacts.py index 5de9eb6..9cf91e0 100644 --- a/scripts/clean_artifacts.py +++ b/scripts/clean_artifacts.py @@ -3,12 +3,25 @@ from pathlib import Path -def main(): +def _clean_dir(target: Path) -> None: + """Wipe non-.dvc files from a directory and remove empty subdirs.""" + print(f"Cleaning directory: {target.name}/ (preserving .dvc files)") + for path in target.rglob("*"): + if path.is_file() and path.suffix != ".dvc": + try: + path.unlink() + except Exception as e: + print(f" Failed delete {path}: {e}") + for path in sorted(target.rglob("*"), key=lambda p: len(p.parts), reverse=True): + if path.is_dir() and not any(path.iterdir()): + path.rmdir() + + +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--yes", action="store_true", help="Skip confirmation prompt") args = parser.parse_args() - # Dynamically resolve project root regardless of where script is called project_root = Path(__file__).resolve().parents[1] if not args.yes: @@ -19,33 +32,15 @@ def main(): print("Aborted.") return - dirs_to_process = ["mlruns", "mlartifacts", "models"] - files_to_delete = ["mlflow.db"] # Specifically target this outside if needed - - for d in dirs_to_process: + for d in ["mlruns", "mlartifacts", "models"]: target = project_root / d if target.exists() and target.is_dir(): - print(f"Cleaning directory: {d}/ (preserving .dvc files)") - # Recursive cleanup - for path in target.rglob("*"): - if path.is_file() and path.suffix != ".dvc": - try: - path.unlink() - # print(f" Deleted: {path.relative_to(project_root)}") - except Exception as e: - print(f" Failed delete {path}: {e}") - - # Clean up empty subdirectories (except DVC metadata folders if any) - # We walk bottom-up to remove childless dirs - for path in sorted(target.rglob("*"), key=lambda p: len(p.parts), reverse=True): - if path.is_dir() and not any(path.iterdir()): - path.rmdir() - - for f in files_to_delete: - target = project_root / f - if target.exists() and target.is_file(): - os.remove(target) - print(f"Deleted root-level file: {f}") + _clean_dir(target) + + db = project_root / "mlflow.db" + if db.exists(): + os.remove(db) + print("Deleted root-level file: mlflow.db") print("Wiped artifacts successfully.") diff --git a/scripts/export_model.py b/scripts/export_model.py index d8c8f88..1a4635b 100644 --- a/scripts/export_model.py +++ b/scripts/export_model.py @@ -13,17 +13,21 @@ logger = logging.getLogger(__name__) -def export_latest_model(): +def export_latest_model() -> None: """ - LOCAL UTILITY — not called by the CI/CD pipeline. + LOCAL UTILITY — not called by the CI/CD pipeline or 'just deploy-all'. Finds the latest run in 'Rossmann_Production' and copies the model artifact from the MLflow artifact store to models/production_model/. Use this locally to swap in a specific historical run without retraining: - MLFLOW_TRACKING_URI=... uv run python scripts/export_model.py + uv run python -m scripts.export_model - In CI, train_model.py saves the model directly to models/ during training. + NOTE: DagsHub runs an MLflow 2.x server which does not support model + artifact storage from MLflow 3.x clients (artifact_path upload silently + fails). Model artifacts are versioned via DVC instead. Run 'just pull' + to restore the latest DVC-tracked model, or 'just train-prod' to train + a fresh one locally. """ mlflow.set_tracking_uri( os.getenv("MLFLOW_TRACKING_URI", "sqlite:///mlruns/mlflow.db") diff --git a/src/rossmann_ops/train_model.py b/src/rossmann_ops/train_model.py index c6a10d1..e309c50 100644 --- a/src/rossmann_ops/train_model.py +++ b/src/rossmann_ops/train_model.py @@ -209,7 +209,7 @@ def train_production_model() -> None: mlflow.log_artifact(shap_tmp) # 13. Log Model to MLflow Registry - mlflow.sklearn.log_model(model, name="production_model") + mlflow.sklearn.log_model(model, artifact_path="production_model") # 14. Save Model Locally for Docker/CI builds local_model_dir = project_root / config["model"]["save_path"]