diff --git a/README.md b/README.md index 8f44e41..f4f6d43 100644 --- a/README.md +++ b/README.md @@ -47,17 +47,19 @@ _Columns: Input images, State-of-the-art predictions, CABiNet predictions (white CABiNet and Ultralytics YOLO26 (`semantic` task, dense per-pixel — not `-seg`) are trained and evaluated under one shared UAVid pipeline (`images/`+`masks/` format, 8 classes incl. Clutter). -| Model | mIoU (%) | Params (M) | FLOPs (B) | HF Weights | -| --------------------------- | -------- | ---------- | --------- | ----------------------------------------------------------------------------- | -| CABiNet (MobileNetV3-Large) | TBD¹ | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/cabinet-mobilenetv3-small-uavid) | -| CABiNet (MobileNetV3-Small) | TBD² | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/cabinet-mobilenetv3-large-uavid) | -| YOLO26n-sem | 63.58 | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26n-sem) | -| YOLO26s-sem | 66.88 | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26s-sem) | -| YOLO26m-sem | 67.66 | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26m-sem) | -| YOLO26l-sem | 67.20 | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26l-sem) | -| YOLO26x-sem | TBD³ | TBD | TBD | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26x-sem) | - -¹ Trained under the refactored pipeline but not yet re-evaluated. ² Not yet trained. ³ Not yet trained. +All numbers below are test-split mIoU (not val — see [`train_yolo.py`](src/scripts/train_yolo.py)'s `validation_config.split`). Params/FLOPs are architecture-only, measured at 1024×1024 via `thop` (CABiNet) / Ultralytics' built-in profiler (YOLO26, also `thop`-backed — both report FLOPs as 2× MACs, so the two families are directly comparable). + +| Model | mIoU (%) | Params (M) | FLOPs (GFLOPs) | HF Weights | +| --------------------------- | -------- | ---------- | -------------- | ----------------------------------------------------------------------------- | +| CABiNet (MobileNetV3-Large) | 68.60 | 9.17 | 54.8 | [HF Model](https://huggingface.co/dronefreak/cabinet-mobilenetv3-large-uavid) | +| CABiNet (MobileNetV3-Small) | 66.84 | 5.36 | 44.1 | [HF Model](https://huggingface.co/dronefreak/cabinet-mobilenetv3-small-uavid) | +| YOLO26x-sem | 64.41 | 40.16 | 430.9 | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26x-sem) | +| YOLO26l-sem | 63.28 | 17.87 | 192.4 | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26l-sem) | +| YOLO26m-sem | 61.98 | 14.32 | 152.3 | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26m-sem) | +| YOLO26s-sem | 61.69 | 6.50 | 44.4 | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26s-sem) | +| YOLO26n-sem | 58.17 | 1.63 | 11.4 | [HF Model](https://huggingface.co/dronefreak/uavid-yolo26n-sem) | + +CABiNet (both backbones) outperforms **every** YOLO26-sem variant on UAVid, including the largest (YOLO26x) — and does so at a fraction of the compute: CABiNet-Large (54.8 GFLOPs) beats YOLO26x (430.9 GFLOPs, ~8× more) and YOLO26m (152.3 GFLOPs, ~3× more) on both mIoU and FLOPs simultaneously. CABiNet-Small also improves substantially over the numbers originally reported in the [CABiNet paper](#citation) on this dataset. ## Installation @@ -422,7 +424,7 @@ python src/scripts/visualize.py ### Performance Profiling -Benchmark inference timing and memory usage (`profile_model_flops` currently measures `torch.profiler` CPU/CUDA time, not analytic FLOP counts — real FLOPs for the [UAVid Model Zoo](#uavid-model-zoo) table are still TBD): +Benchmark inference timing and memory usage (`profile_model_flops` measures `torch.profiler` CPU/CUDA time, not analytic FLOP counts — the analytic Params/FLOPs in the [UAVid Model Zoo](#uavid-model-zoo) table above were computed separately via `thop`): ```python from src.utils.profiler import PerformanceProfiler diff --git a/configs/train_yolo.yaml b/configs/train_yolo.yaml index daa731e..0a7e1fe 100644 --- a/configs/train_yolo.yaml +++ b/configs/train_yolo.yaml @@ -115,6 +115,7 @@ validation_config: save_json: true augment: false # TTA: set true on CLI for augmented eval weights: ~ # null → auto-inferred from experiments_path/model.run_name + split: val # 'val' or 'test' — override: validation_config.split=test # ── Runtime ────────────────────────────────────────────────────────────────── runtime: diff --git a/src/scripts/train_yolo.py b/src/scripts/train_yolo.py index 6a84575..63a83e2 100644 --- a/src/scripts/train_yolo.py +++ b/src/scripts/train_yolo.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- encoding: utf-8 -*- +import json import logging from pathlib import Path @@ -11,6 +12,21 @@ logger = logging.getLogger(__name__) +# UAVid classes in trainId order (0-7) — matches configs/UAVid_info.json and +# hf_modelcards/generate_hf_model_zoo.py's UAVID_CLASSES. Used only to label +# mode=val's per-class IoU printout in a form directly pastable into a +# hf_modelcards/model_metrics//metrics.json. +UAVID_CLASS_NAMES = [ + "Clutter", + "Building", + "Road", + "Static Car", + "Tree", + "Vegetation", + "Human", + "Moving Car", +] + # Repo root: src/scripts/train_yolo.py -> src/scripts -> src -> REPO_ROOT = Path(__file__).resolve().parents[2] @@ -164,7 +180,7 @@ def _build_val_kwargs(cfg: DictConfig, dataset_path: Path, weights: str) -> dict "imgsz": int(cfg.training_config.imgsz), "batch": int(vc.get("batch_size", 1)), "device": cfg.runtime.get("device", 0), - "split": "val", + "split": str(vc.get("split", "val")), "save_json": bool(vc.get("save_json", True)), "augment": bool(vc.get("augment", False)), "plots": True, @@ -239,12 +255,34 @@ def main(cfg: DictConfig) -> None: console.print(f"Validating with weights: {weights_path}", style="info") model = YOLO(str(weights_path)) val_kwargs = _build_val_kwargs(cfg, dataset_path, str(weights_path)) + split = val_kwargs["split"] results = model.val(**val_kwargs) if results is not None: + console.print(f"Split: {split}", style="info") console.print(f"mIoU: {results.miou:.4f}", style="info") console.print(f"Pixel accuracy: {results.pixel_accuracy:.4f}", style="info") + per_class_iou = { + name: round(float(iou), 4) + for name, iou in zip(UAVID_CLASS_NAMES, results.per_class_iou) + } + console.print("Per-class IoU:", style="info") + for name, iou in per_class_iou.items(): + console.print(f" {name}: {iou:.4f}", style="info") + + metrics_snippet = { + "eval_split": split, + "miou": round(float(results.miou), 4), + "pixel_acc": round(float(results.pixel_accuracy), 4), + "per_class_iou": per_class_iou, + } + console.print( + "\nPaste into hf_modelcards/model_metrics//metrics.json:", + style="info", + ) + console.print(json.dumps(metrics_snippet, indent=2), style="info") + else: raise ValueError( f"Unknown mode '{mode}'. Choose 'train' or 'val'.\n"