From 213540e0f88f12fe7dd503d4e3368cb9ee6fe7df Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 10 Oct 2018 22:37:02 +0200
Subject: [PATCH 01/97] Display Hyperparameters weights, only if needed and
 used by the choosen loss

---
 robosat/tools/train.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 85245fd8..3570cb44 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -75,10 +75,10 @@ def main(args):
     if model["common"]["cuda"]:
         torch.backends.cudnn.benchmark = True
 
-    try:
-        weight = torch.Tensor(dataset["weights"]["values"])
-    except KeyError:
-        if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
+    if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
+        try:
+            weight = torch.Tensor(dataset["weights"]["values"])
+        except KeyError:
             sys.exit("Error: The loss function used, need dataset weights values")
 
     optimizer = Adam(net.parameters(), lr=model["opt"]["lr"], weight_decay=model["opt"]["decay"])

From 6991719cb72ee9a166ebb1f778284b25ab115d92 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 12 Oct 2018 11:45:33 +0200
Subject: [PATCH 02/97] Data Augmentation: Add upscale_factor, remove useless
 CenterCrop

---
 config/model-unet.toml | 3 +++
 robosat/tools/train.py | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 6effd699..4faba868 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -32,3 +32,6 @@
 
   # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
   loss = 'Lovasz'
+
+  # Upscale input value factor to enhance resolution (classical values: 1, 2 or 4)
+  upscale_factor = 1
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 3570cb44..afeb6dc9 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -248,6 +248,7 @@ def validate(loader, num_classes, device, net, criterion):
 
 def get_dataset_loaders(model, dataset, workers):
     target_size = (model["common"]["image_size"],) * 2
+    target_size = tuple([size * model["opt"]["upscale_factor"] for size in target_size])
     batch_size = model["common"]["batch_size"]
     path = dataset["common"]["dataset"]
 
@@ -257,7 +258,6 @@ def get_dataset_loaders(model, dataset, workers):
         [
             JointTransform(ConvertImageMode("RGB"), ConvertImageMode("P")),
             JointTransform(Resize(target_size, Image.BILINEAR), Resize(target_size, Image.NEAREST)),
-            JointTransform(CenterCrop(target_size), CenterCrop(target_size)),
             JointRandomHorizontalFlip(0.5),
             JointRandomRotation(0.5, 90),
             JointRandomRotation(0.5, 90),

From bbd86a358a903114f842d61d17467d35fd292cbe Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 13 Oct 2018 14:33:25 +0200
Subject: [PATCH 03/97] Add in rs download, TMS and WMS services support.
 Expose timeout in userland

---
 robosat/tiles.py          | 17 +++++++++++++++++
 robosat/tools/download.py | 21 ++++++++++++++++-----
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index 24a09a1e..09e98877 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -13,6 +13,7 @@
 import os
 
 from PIL import Image
+from pyproj import Proj, transform
 import mercantile
 
 
@@ -42,6 +43,22 @@ def lerp(a, b, c):
     return lon, lat
 
 
+def tile_to_bbox(tile):
+    """Convert a tile to bbox coordinates
+
+    Args:
+      tile: the mercantile tile
+
+    Returns:
+       Tile's bbox coordinates (expressed in EPSG:3857)
+    """
+
+    west, south, east, north = mercantile.bounds(tile)
+    x, y = transform(Proj("+init=EPSG:4326"), Proj("+init=EPSG:3857"), [west, east], [north, south])
+
+    return [min(x), min(y), max(x), max(y)]
+
+
 def fetch_image(session, url, timeout=10):
     """Fetches the image representation for a tile.
 
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index b6ddf5a6..a69831c9 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -8,17 +8,21 @@
 from PIL import Image
 from tqdm import tqdm
 
-from robosat.tiles import tiles_from_csv, fetch_image
+from robosat.tiles import tiles_from_csv, fetch_image, tile_to_bbox
 
 
 def add_parser(subparser):
     parser = subparser.add_parser(
-        "download", help="downloads images from Mapbox Maps API", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        "download", help="downloads images from a remote server", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("url", type=str, help="endpoint with {z}/{x}/{y} variables to fetch image tiles from")
+    parser.add_argument(
+        "url", type=str, help="endpoint with {z}/{x}/{y} or {xmin},{ymin},{xmax},{ymax} variables to fetch image tiles"
+    )
     parser.add_argument("--ext", type=str, default="webp", help="file format to save images in")
     parser.add_argument("--rate", type=int, default=10, help="rate limit in max. requests per second")
+    parser.add_argument("--type", type=str, default="XYZ", help="service type to use (e.g: XYZ, WMS or TMS)")
+    parser.add_argument("--timeout", type=int, default=10, help="server request timeout (in seconds)")
     parser.add_argument("tiles", type=str, help="path to .csv tiles file")
     parser.add_argument("out", type=str, help="path to slippy map directory for storing tiles")
 
@@ -48,9 +52,16 @@ def worker(tile):
                 if os.path.isfile(path):
                     return tile, True
 
-                url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
+                if args.type == "XYZ":
+                    url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
+                elif args.type == "TMS":
+                    tile.y = (2 ** tile.z) - tile.y - 1
+                    url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
+                elif args.type == "WMS":
+                    xmin, ymin, xmax, ymax = tile_to_bbox(tile)
+                    url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
 
-                res = fetch_image(session, url)
+                res = fetch_image(session, url, args.timeout)
 
                 if not res:
                     return tile, False

From 9c9b307cf20956768479b4b62473f8eb1c309f26 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 14 Oct 2018 14:13:01 +0200
Subject: [PATCH 04/97] Homogenize reprojection calls, pass throught rasterio

---
 robosat/tiles.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index 09e98877..93bad0ea 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -13,7 +13,8 @@
 import os
 
 from PIL import Image
-from pyproj import Proj, transform
+from rasterio.warp import transform
+from rasterio.crs import CRS
 import mercantile
 
 
@@ -54,7 +55,7 @@ def tile_to_bbox(tile):
     """
 
     west, south, east, north = mercantile.bounds(tile)
-    x, y = transform(Proj("+init=EPSG:4326"), Proj("+init=EPSG:3857"), [west, east], [north, south])
+    x, y = transform(CRS.from_epsg(4326), CRS.from_epsg(3857), [west, east], [north, south])
 
     return [min(x), min(y), max(x), max(y)]
 

From d51393dfe07c7b241ea2442dc460b3d257bf1197 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 15 Oct 2018 15:13:59 +0200
Subject: [PATCH 05/97] Add GeoJSON MultiPolygon support

---
 robosat/tools/rasterize.py | 48 ++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 28 deletions(-)

diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 4b9abbb8..30ce1c01 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -14,6 +14,7 @@
 from rasterio.features import rasterize
 from rasterio.warp import transform
 from supermercado import burntiles
+from shapely.geometry import shape, mapping
 
 from robosat.config import load_config
 from robosat.colors import make_palette
@@ -36,46 +37,34 @@ def add_parser(subparser):
 
 
 def feature_to_mercator(feature):
-    """Normalize feature and converts coords to 3857.
+    """Convert polygon feature coords to 3857.
 
     Args:
       feature: geojson feature to convert to mercator geometry.
     """
     # Ref: https://gist.github.com/dnomadb/5cbc116aacc352c7126e779c29ab7abe
 
-    src_crs = CRS.from_epsg(4326)
-    dst_crs = CRS.from_epsg(3857)
-
-    geometry = feature["geometry"]
-    if geometry["type"] == "Polygon":
-        xys = (zip(*part) for part in geometry["coordinates"])
-        xys = (list(zip(*transform(src_crs, dst_crs, *xy))) for xy in xys)
+    if feature["geometry"]["type"] == "Polygon":
+        xys = (zip(*ring) for ring in feature["geometry"]["coordinates"])
+        xys = (list(zip(*transform(CRS.from_epsg(4326), CRS.from_epsg(3857), *xy))) for xy in xys)
 
         yield {"coordinates": list(xys), "type": "Polygon"}
 
-    elif geometry["type"] == "MultiPolygon":
-        for component in geometry["coordinates"]:
-            xys = (zip(*part) for part in component)
-            xys = (list(zip(*transform(src_crs, dst_crs, *xy))) for xy in xys)
-
-            yield {"coordinates": list(xys), "type": "Polygon"}
 
-
-def burn(tile, features, size):
+def burn(tile, features, size, burn_value=1):
     """Burn tile with features.
 
     Args:
       tile: the mercantile tile to burn.
       features: the geojson features to burn.
       size: the size of burned image.
+      burn_value: the value you want in the output raster where a shape exists
 
     Returns:
       image: rasterized file of size with features burned.
     """
 
-    # the value you want in the output raster where a shape exists
-    burnval = 1
-    shapes = ((geometry, burnval) for feature in features for geometry in feature_to_mercator(feature))
+    shapes = ((geometry, burn_value) for feature in features for geometry in feature_to_mercator(feature))
 
     bounds = mercantile.xy_bounds(tile)
     transform = from_bounds(*bounds, size, size)
@@ -107,15 +96,18 @@ def main(args):
     feature_map = collections.defaultdict(list)
     for i, feature in enumerate(tqdm(fc["features"], ascii=True, unit="feature")):
 
-        if feature["geometry"]["type"] != "Polygon":
-            continue
-
-        try:
-            for tile in burntiles.burn([feature], zoom=args.zoom):
-                feature_map[mercantile.Tile(*tile)].append(feature)
-        except ValueError as e:
-            print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr)
-            continue
+        if feature["geometry"]["type"] == "Polygon":
+            feature["geometry"]["coordinates"] = [feature["geometry"]["coordinates"]]
+            feature["geometry"]["type"] = "MultiPolygon"
+
+        for polygon in shape(feature["geometry"]):
+            simple_feature = {"type": "feature", "geometry": mapping(polygon)}
+            try:
+                for tile in burntiles.burn([simple_feature], zoom=args.zoom):
+                    feature_map[mercantile.Tile(*tile)].append(simple_feature)
+            except ValueError as e:
+                print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr)
+                continue
 
     # Burn features to tiles and write to a slippy map directory.
     for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit="tile"):

From d8f1686baf3a001a4997bd2b475c5494681b2479 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 16 Oct 2018 00:54:04 +0200
Subject: [PATCH 06/97] Add url in user error message. Neat for WMS use cases

---
 robosat/tools/download.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index a69831c9..1be70ff2 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -50,7 +50,7 @@ def worker(tile):
                 path = os.path.join(args.out, z, x, "{}.{}".format(y, args.ext))
 
                 if os.path.isfile(path):
-                    return tile, True
+                    return tile, None, True
 
                 if args.type == "XYZ":
                     url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
@@ -64,13 +64,13 @@ def worker(tile):
                 res = fetch_image(session, url, args.timeout)
 
                 if not res:
-                    return tile, False
+                    return tile, url, False
 
                 try:
                     image = Image.open(res)
                     image.save(path, optimize=True)
                 except OSError:
-                    return tile, False
+                    return tile, url, False
 
                 tock = time.monotonic()
 
@@ -82,8 +82,8 @@ def worker(tile):
 
                 progress.update()
 
-                return tile, True
+                return tile, url, True
 
-            for tile, ok in executor.map(worker, tiles):
+            for tile, url, ok in executor.map(worker, tiles):
                 if not ok:
-                    print("Warning: {} failed, skipping".format(tile), file=sys.stderr)
+                    print("Warning:\n {} failed, skipping.\n {}\n".format(tile, url), file=sys.stderr)

From 43a605086a38e0c3663315a1420436a6765fb9a0 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 16 Oct 2018 01:10:33 +0200
Subject: [PATCH 07/97] Change image_upscale hyperparameter name and add entry
 in log

---
 config/model-unet.toml | 4 ++--
 robosat/tools/train.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 4faba868..01e3d292 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -33,5 +33,5 @@
   # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
   loss = 'Lovasz'
 
-  # Upscale input value factor to enhance resolution (classical values: 1, 2 or 4)
-  upscale_factor = 1
+  # Upscale factor to input image resolution (classical values: 1, 2 or 4) 
+  image_upscale = 1
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index afeb6dc9..83e97442 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -120,6 +120,7 @@ def map_location(storage, _):
     log.log("--- Hyper Parameters on Dataset: {} ---".format(dataset["common"]["dataset"]))
     log.log("Batch Size:\t {}".format(model["common"]["batch_size"]))
     log.log("Image Size:\t {}".format(model["common"]["image_size"]))
+    log.log("Image Upscale:\t {}".format(model["opt"]["image_upscale"]))
     log.log("Learning Rate:\t {}".format(model["opt"]["lr"]))
     log.log("Weight Decay:\t {}".format(model["opt"]["decay"]))
     log.log("Loss function:\t {}".format(model["opt"]["loss"]))
@@ -248,7 +249,7 @@ def validate(loader, num_classes, device, net, criterion):
 
 def get_dataset_loaders(model, dataset, workers):
     target_size = (model["common"]["image_size"],) * 2
-    target_size = tuple([size * model["opt"]["upscale_factor"] for size in target_size])
+    target_size = tuple([size * model["opt"]["image_upscale"] for size in target_size])
     batch_size = model["common"]["batch_size"]
     path = dataset["common"]["dataset"]
 

From b20e5a42a0da32a747b6cb5d5e67a7a484b917f0 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Thu, 18 Oct 2018 08:47:59 +0200
Subject: [PATCH 08/97] Fix mIoU to be 0 div resilient

---
 robosat/metrics.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/robosat/metrics.py b/robosat/metrics.py
index 5d125e2a..3b523adf 100644
--- a/robosat/metrics.py
+++ b/robosat/metrics.py
@@ -46,7 +46,13 @@ def get_miou(self):
         Returns:
           The mean Intersection over Union score for all observations seen so far.
         """
-        return np.nanmean([self.tn / (self.tn + self.fn + self.fp), self.tp / (self.tp + self.fn + self.fp)])
+
+        try:
+            miou = np.nanmean([self.tn / (self.tn + self.fn + self.fp), self.tp / (self.tp + self.fn + self.fp)])
+        except ZeroDivisionError:
+            miou = float("Inf")
+
+        return miou
 
     def get_fg_iou(self):
         """Retrieves the foreground Intersection over Union score.

From 73933fde7bb9f2c4ef2792e94e00341d8cd1b15a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Thu, 18 Oct 2018 16:17:57 +0200
Subject: [PATCH 09/97] Move to PIL Image to OpenCV H,W,C RGB for
 DataAugmentation. Refactor Data Augmentation

---
 config/model-unet.toml   |   7 +-
 robosat/datasets.py      |  37 ++++----
 robosat/tools/predict.py |   4 +-
 robosat/tools/serve.py   |   7 +-
 robosat/tools/train.py   |  42 +++++----
 robosat/tools/weights.py |   6 +-
 robosat/transforms.py    | 180 ++++++++++++++++-----------------------
 tests/test_datasets.py   |   9 +-
 8 files changed, 130 insertions(+), 162 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 01e3d292..44659055 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -33,5 +33,8 @@
   # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
   loss = 'Lovasz'
 
-  # Upscale factor to input image resolution (classical values: 1, 2 or 4) 
-  image_upscale = 1
+  # Resize factor to input image resolution
+  image_resize_factor = 1
+
+  # Data augmentation, Flip or Rotate probabilty
+  flip_or_rotate_prob = 0.25
diff --git a/robosat/datasets.py b/robosat/datasets.py
index 80659679..2b1e4f2d 100644
--- a/robosat/datasets.py
+++ b/robosat/datasets.py
@@ -8,6 +8,8 @@
 import torch
 from PIL import Image
 import torch.utils.data
+import cv2
+import numpy as np
 
 from robosat.tiles import tiles_from_slippy_map, buffer_tile_image
 
@@ -17,7 +19,7 @@ class SlippyMapTiles(torch.utils.data.Dataset):
     """Dataset for images stored in slippy map format.
     """
 
-    def __init__(self, root, transform=None):
+    def __init__(self, root, mode, transform=None):
         super().__init__()
 
         self.tiles = []
@@ -25,13 +27,19 @@ def __init__(self, root, transform=None):
 
         self.tiles = [(tile, path) for tile, path in tiles_from_slippy_map(root)]
         self.tiles.sort(key=lambda tile: tile[0])
+        self.mode = mode
 
     def __len__(self):
         return len(self.tiles)
 
     def __getitem__(self, i):
         tile, path = self.tiles[i]
-        image = Image.open(path)
+
+        if self.mode == "image":
+            image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
+
+        elif self.mode == "mask":
+            image = np.array(Image.open(path).convert("P"))
 
         if self.transform is not None:
             image = self.transform(image)
@@ -40,42 +48,35 @@ def __getitem__(self, i):
 
 
 # Multiple Slippy Map directories.
-# Think: one with images, one with masks, one with rasterized traces.
 class SlippyMapTilesConcatenation(torch.utils.data.Dataset):
     """Dataset to concate multiple input images stored in slippy map format.
     """
 
-    def __init__(self, inputs, target, joint_transform=None):
+    def __init__(self, input, target, joint_transform=None):
         super().__init__()
 
         # No transformations in the `SlippyMapTiles` instead joint transformations in getitem
         self.joint_transform = joint_transform
 
-        self.inputs = [SlippyMapTiles(inp) for inp in inputs]
-        self.target = SlippyMapTiles(target)
+        self.target = SlippyMapTiles(target, mode="mask")
+        self.input = SlippyMapTiles(input, mode="image")
 
-        assert len(set([len(dataset) for dataset in self.inputs])) == 1, "same number of tiles in all images"
-        assert len(self.target) == len(self.inputs[0]), "same number of tiles in images and label"
+        assert len(self.input) == len(self.target), "same number of tiles in images and label"
 
     def __len__(self):
         return len(self.target)
 
     def __getitem__(self, i):
-        # at this point all transformations are applied and we expect to work with raw tensors
-        inputs = [dataset[i] for dataset in self.inputs]
-
-        images = [image for image, _ in inputs]
-        tiles = [tile for _, tile in inputs]
 
+        image, image_tile = self.input[i]
         mask, mask_tile = self.target[i]
 
-        assert len(set(tiles)) == 1, "all images are for the same tile"
-        assert tiles[0] == mask_tile, "image tile is the same as label tile"
+        assert image_tile == mask_tile, "image tile is the same as label tile"
 
         if self.joint_transform is not None:
-            images, mask = self.joint_transform(images, mask)
+            image, mask = self.joint_transform(image, mask)
 
-        return torch.cat(images, dim=0), mask, tiles
+        return image, mask, image_tile[0]
 
 
 # Todo: once we have the SlippyMapDataset this dataset should wrap
@@ -113,7 +114,7 @@ def __len__(self):
 
     def __getitem__(self, i):
         tile, path = self.tiles[i]
-        image = buffer_tile_image(tile, self.tiles, overlap=self.overlap, tile_size=self.size)
+        image = np.array(buffer_tile_image(tile, self.tiles, overlap=self.overlap, tile_size=self.size))
 
         if self.transform is not None:
             image = self.transform(image)
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 36436b52..88edcc50 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -17,7 +17,7 @@
 from robosat.unet import UNet
 from robosat.config import load_config
 from robosat.colors import continuous_palette_for_color
-from robosat.transforms import ConvertImageMode, ImageToTensor
+from robosat.transforms import ImageToTensor
 
 
 def add_parser(subparser):
@@ -70,7 +70,7 @@ def map_location(storage, _):
 
     mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
 
-    transform = Compose([ConvertImageMode(mode="RGB"), ImageToTensor(), Normalize(mean=mean, std=std)])
+    transform = Compose([ImageToTensor(), Normalize(mean=mean, std=std)])
 
     directory = BufferedSlippyMapDirectory(args.tiles, transform=transform, size=args.tile_size, overlap=args.overlap)
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
diff --git a/robosat/tools/serve.py b/robosat/tools/serve.py
index a3e6252c..2bc391f4 100644
--- a/robosat/tools/serve.py
+++ b/robosat/tools/serve.py
@@ -12,6 +12,7 @@
 
 import mercantile
 import requests
+import cv2
 from PIL import Image
 from flask import Flask, send_file, render_template, abort
 
@@ -19,7 +20,7 @@
 from robosat.unet import UNet
 from robosat.config import load_config
 from robosat.colors import make_palette
-from robosat.transforms import ConvertImageMode, ImageToTensor
+from robosat.transforms import ImageToTensor
 
 """
 Simple tile server running a segmentation model on the fly.
@@ -62,7 +63,7 @@ def tile(z, x, y):
     if not res:
         abort(500)
 
-    image = Image.open(res)
+    image = cv2.cvtColor(cv2.imread(res), cv2.COLOR_BGR2RGB)
 
     mask = predictor.segment(image)
 
@@ -152,7 +153,7 @@ def segment(self, image):
         with torch.no_grad():
             mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
 
-            transform = Compose([ConvertImageMode(mode="RGB"), ImageToTensor(), Normalize(mean=mean, std=std)])
+            transform = Compose([ImageToTensor(), Normalize(mean=mean, std=std)])
             image = transform(image)
 
             batch = image.unsqueeze(0).to(self.device)
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 83e97442..689bdd44 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -11,16 +11,15 @@
 from torch.nn import DataParallel
 from torch.optim import Adam
 from torch.utils.data import DataLoader
-from torchvision.transforms import Resize, CenterCrop, Normalize
+from torchvision.transforms import Normalize
 
 from tqdm import tqdm
 
 from robosat.transforms import (
     JointCompose,
     JointTransform,
-    JointRandomHorizontalFlip,
-    JointRandomRotation,
-    ConvertImageMode,
+    JointResize,
+    JointRandomFlipOrRotate,
     ImageToTensor,
     MaskToTensor,
 )
@@ -118,14 +117,15 @@ def map_location(storage, _):
     log = Log(os.path.join(model["common"]["checkpoint"], "log"))
 
     log.log("--- Hyper Parameters on Dataset: {} ---".format(dataset["common"]["dataset"]))
-    log.log("Batch Size:\t {}".format(model["common"]["batch_size"]))
-    log.log("Image Size:\t {}".format(model["common"]["image_size"]))
-    log.log("Image Upscale:\t {}".format(model["opt"]["image_upscale"]))
-    log.log("Learning Rate:\t {}".format(model["opt"]["lr"]))
-    log.log("Weight Decay:\t {}".format(model["opt"]["decay"]))
-    log.log("Loss function:\t {}".format(model["opt"]["loss"]))
+    log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
+    log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
+    log.log("Image Resize factor:\t {}".format(model["opt"]["image_resize_factor"]))
+    log.log("Flip or Rotate prob:\t {}".format(model["opt"]["flip_or_rotate_prob"]))
+    log.log("Learning Rate:\t\t {}".format(model["opt"]["lr"]))
+    log.log("Weight Decay:\t\t {}".format(model["opt"]["decay"]))
+    log.log("Loss function:\t\t {}".format(model["opt"]["loss"]))
     if "weight" in locals():
-        log.log("Weights :\t {}".format(dataset["weights"]["values"]))
+        log.log("Weights :\t\t {}".format(dataset["weights"]["values"]))
     log.log("---")
 
     for epoch in range(resume, num_epochs):
@@ -248,32 +248,28 @@ def validate(loader, num_classes, device, net, criterion):
 
 
 def get_dataset_loaders(model, dataset, workers):
-    target_size = (model["common"]["image_size"],) * 2
-    target_size = tuple([size * model["opt"]["image_upscale"] for size in target_size])
-    batch_size = model["common"]["batch_size"]
-    path = dataset["common"]["dataset"]
 
+    # Values computed on ImageNet DataSet
     mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
 
     transform = JointCompose(
         [
-            JointTransform(ConvertImageMode("RGB"), ConvertImageMode("P")),
-            JointTransform(Resize(target_size, Image.BILINEAR), Resize(target_size, Image.NEAREST)),
-            JointRandomHorizontalFlip(0.5),
-            JointRandomRotation(0.5, 90),
-            JointRandomRotation(0.5, 90),
-            JointRandomRotation(0.5, 90),
+            JointResize(model["opt"]["image_resize_factor"]),
+            JointRandomFlipOrRotate(model["opt"]["flip_or_rotate_prob"]),
             JointTransform(ImageToTensor(), MaskToTensor()),
             JointTransform(Normalize(mean=mean, std=std), None),
         ]
     )
 
+    batch_size = model["common"]["batch_size"]
+    path = dataset["common"]["dataset"]
+
     train_dataset = SlippyMapTilesConcatenation(
-        [os.path.join(path, "training", "images")], os.path.join(path, "training", "labels"), transform
+        os.path.join(path, "training", "images"), os.path.join(path, "training", "labels"), transform
     )
 
     val_dataset = SlippyMapTilesConcatenation(
-        [os.path.join(path, "validation", "images")], os.path.join(path, "validation", "labels"), transform
+        os.path.join(path, "validation", "images"), os.path.join(path, "validation", "labels"), transform
     )
 
     train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=workers)
diff --git a/robosat/tools/weights.py b/robosat/tools/weights.py
index c154fffd..9caf11e2 100644
--- a/robosat/tools/weights.py
+++ b/robosat/tools/weights.py
@@ -10,7 +10,7 @@
 
 from robosat.config import load_config
 from robosat.datasets import SlippyMapTiles
-from robosat.transforms import ConvertImageMode, MaskToTensor
+from robosat.transforms import MaskToTensor
 
 
 def add_parser(subparser):
@@ -29,9 +29,9 @@ def main(args):
     path = dataset["common"]["dataset"]
     num_classes = len(dataset["common"]["classes"])
 
-    train_transform = Compose([ConvertImageMode(mode="P"), MaskToTensor()])
+    train_transform = Compose([MaskToTensor()])
 
-    train_dataset = SlippyMapTiles(os.path.join(path, "training", "labels"), transform=train_transform)
+    train_dataset = SlippyMapTiles(os.path.join(path, "training", "labels"), "mask", transform=train_transform)
 
     n = 0
     counts = np.zeros(num_classes, dtype=np.int64)
diff --git a/robosat/transforms.py b/robosat/transforms.py
index 9347fa34..3c18c982 100644
--- a/robosat/transforms.py
+++ b/robosat/transforms.py
@@ -2,56 +2,43 @@
 """
 
 import random
-
 import torch
+import cv2
 import numpy as np
-from PIL import Image
-
-import torchvision
-
 
-# Callable to convert a RGB image into a PyTorch tensor.
-ImageToTensor = torchvision.transforms.ToTensor
 
-
-class MaskToTensor:
-    """Callable to convert a PIL image into a PyTorch tensor.
+class ImageToTensor:
+    """Callable to convert a OpenCV H,W,C image into a PyTorch tensor.
     """
 
     def __call__(self, image):
         """Converts the image into a tensor.
 
         Args:
-          image: the PIL image to convert into a PyTorch tensor.
+          image: the image to convert into a PyTorch tensor.
 
         Returns:
           The converted PyTorch tensor.
         """
 
-        return torch.from_numpy(np.array(image, dtype=np.uint8)).long()
+        return torch.from_numpy(np.moveaxis(image, 2, 0)).float()
 
 
-class ConvertImageMode:
-    """Callable to convert a PIL image into a specific image mode (e.g. RGB, P)
+class MaskToTensor:
+    """Callable to convert an OpenCV H,W image into a PyTorch tensor.
     """
 
-    def __init__(self, mode):
-        """Creates an `ConvertImageMode` instance.
+    def __call__(self, tensor):
+        """Converts the image into a tensor.
 
         Args:
-          mode: the PIL image mode string
-        """
-
-        self.mode = mode
-
-    def __call__(self, image):
-        """Applies to mode conversion to an image.
+          image: the image to convert into a PyTorch tensor.
 
-        Args:
-          image: the PIL.Image image to transform.
+        Returns:
+          The converted PyTorch tensor.
         """
 
-        return image.convert(self.mode)
+        return torch.from_numpy(tensor).long()
 
 
 class JointCompose:
@@ -67,21 +54,21 @@ def __init__(self, transforms):
 
         self.transforms = transforms
 
-    def __call__(self, images, mask):
-        """Applies multiple transformations to the images and the mask at the same time.
+    def __call__(self, image, mask):
+        """Applies multiple transformations to the image and its mask at the same time.
 
         Args:
-          images: the PIL.Image images to transform.
-          mask: the PIL.Image mask to transform.
+          image: the image to transform.
+          mask: the mask to transform.
 
         Returns:
-          The transformed PIL.Image (images, mask) tuple.
+          The transformed (image, mask) tuple.
         """
 
         for transform in self.transforms:
-            images, mask = transform(images, mask)
+            image, mask = transform(image, mask)
 
-        return images, mask
+        return image, mask
 
 
 class JointTransform:
@@ -94,128 +81,109 @@ def __init__(self, image_transform, mask_transform):
         """Creates an `JointTransform` instance.
 
         Args:
-          image_transform: the transformation to run on the images or `None` for no-op.
+          image_transform: the transformation to run on the image or `None` for no-op.
           mask_transform: the transformation to run on the mask or `None` for no-op.
 
         Returns:
-          The (images, mask) tuple with the transformations applied.
+          The (image, mask) tuple with the transformations applied.
         """
 
         self.image_transform = image_transform
         self.mask_transform = mask_transform
 
-    def __call__(self, images, mask):
-        """Applies the transformations associated with images and their mask.
+    def __call__(self, image, mask):
+        """Applies the transformations associated with image and its mask.
 
         Args:
-          images: the PIL.Image images to transform.
-          mask: the PIL.Image mask to transform.
+          image: the image to transform.
+          mask: the mask to transform.
 
         Returns:
-          The PIL.Image (images, mask) tuple with images and mask transformed.
+          The (image, mask) tuple with the transformations applied.
         """
 
         if self.image_transform is not None:
-            images = [self.image_transform(v) for v in images]
+            image = self.image_transform(image)
 
         if self.mask_transform is not None:
             mask = self.mask_transform(mask)
 
-        return images, mask
+        return image, mask
 
 
-class JointRandomVerticalFlip:
-    """Callable to randomly flip images and its mask top to bottom.
+class JointRandomFlipOrRotate:
+    """Callable to randomly rotate image and its mask.
     """
 
     def __init__(self, p):
-        """Creates an `JointRandomVerticalFlip` instance.
+        """Creates an `JointRandomRotation` instance.
 
         Args:
-          p: the probability for flipping.
+          p: the probability for rotating.
         """
-
+        assert p >= 0.0 and p <= 1.0, "Probability must be expressed in 0-1 interval"
         self.p = p
 
-    def __call__(self, images, mask):
-        """Randomly flips images and their mask top to bottom.
+    def __call__(self, image, mask):
+        """Randomly rotates or flip image and its mask.
 
         Args:
-          images: the PIL.Image image to transform.
-          mask: the PIL.Image mask to transform.
+          image: the image to transform.
+          mask: the mask to transform.
 
         Returns:
-          The PIL.Image (images, mask) tuple with either images and mask flipped or none of them flipped.
-        """
-
-        if random.random() < self.p:
-            return [v.transpose(Image.FLIP_TOP_BOTTOM) for v in images], mask.transpose(Image.FLIP_TOP_BOTTOM)
-        else:
-            return images, mask
-
-
-class JointRandomHorizontalFlip:
-    """Callable to randomly flip images and their mask left to right.
-    """
-
-    def __init__(self, p):
-        """Creates an `JointRandomHorizontalFlip` instance.
-
-        Args:
-          p: the probability for flipping.
+          The (image, mask) tuple with either image and mask flip or rotated or kept untouched (but synced)
         """
 
-        self.p = p
+        if random.random() > self.p:
+            return image, mask
 
-    def __call__(self, images, mask):
-        """Randomly flips image and their mask left to right.
+        transform = random.choice(["Rotate90", "Rotate180", "Rotate270", "HorizontalFlip", "VerticalFlip"])
 
-        Args:
-          images: the PIL.Image images to transform.
-          mask: the PIL.Image mask to transform.
+        if transform == "Rotate90":
+            return cv2.flip(cv2.transpose(image), +1), cv2.flip(cv2.transpose(mask), +1)
+        elif transform == "Rotate180":
+            return cv2.flip(image, -1), cv2.flip(mask, -1)
+        elif transform == "Rotate270":
+            return cv2.flip(cv2.transpose(image), 0), cv2.flip(cv2.transpose(mask), 0)
+        elif transform == "HorizontalFlip":
+            return cv2.flip(image, +1), cv2.flip(mask, +1)
+        elif transform == "VerticalFlip":
+            return cv2.flip(image, 0), cv2.flip(mask, 0)
 
-        Returns:
-          The PIL.Image (images, mask) tuple with either images and mask flipped or none of them flipped.
-        """
 
-        if random.random() < self.p:
-            return [v.transpose(Image.FLIP_LEFT_RIGHT) for v in images], mask.transpose(Image.FLIP_LEFT_RIGHT)
-        else:
-            return images, mask
-
-
-class JointRandomRotation:
-    """Callable to randomly rotate images and their mask.
+class JointResize:
+    """Callable to resize image and its mask
     """
 
-    def __init__(self, p, degree):
-        """Creates an `JointRandomRotation` instance.
+    def __init__(self, f):
+        """Creates an `JointResize` instance.
 
         Args:
-          p: the probability for rotating.
+          f: the desired resize factor
         """
+        assert not (f % 2) or not (int(1 / f) % 2) or f == 1, "Invalid resize factor value"
+        self.f = f
 
-        self.p = p
-
-        methods = {90: Image.ROTATE_90, 180: Image.ROTATE_180, 270: Image.ROTATE_270}
-
-        if degree not in methods.keys():
-            raise NotImplementedError("We only support multiple of 90 degree rotations for now")
-
-        self.method = methods[degree]
-
-    def __call__(self, images, mask):
-        """Randomly rotates images and their mask.
+    def __call__(self, image, mask):
+        """Resize image and its mask
 
         Args:
-          images: the PIL.Image image to transform.
-          mask: the PIL.Image mask to transform.
+          image: the image to transform.
+          mask: the mask to transform.
 
         Returns:
-          The PIL.Image (images, mask) tuple with either images and mask rotated or none of them rotated.
+          The (image, mask) tuple resized
         """
 
-        if random.random() < self.p:
-            return [v.transpose(self.method) for v in images], mask.transpose(self.method)
+        if self.f == 1:
+            return image, mask
+        elif self.f > 1:
+            image_interpolation = cv2.INTER_AREA
         else:
-            return images, mask
+            image_interpolation = cv2.INTER_LINEAR
+
+        return (
+            cv2.resize(image, None, fx=self.f, fy=self.f, interpolation=image_interpolation),
+            cv2.resize(mask, None, fx=self.f, fy=self.f, interpolation=cv2.INTER_NEAREST),
+        )
diff --git a/tests/test_datasets.py b/tests/test_datasets.py
index 97e6aac6..5c19b99f 100644
--- a/tests/test_datasets.py
+++ b/tests/test_datasets.py
@@ -12,17 +12,16 @@ class TestSlippyMapTiles(unittest.TestCase):
     images = "tests/fixtures/images/"
 
     def test_len(self):
-        dataset = SlippyMapTiles(TestSlippyMapTiles.images)
+        dataset = SlippyMapTiles(TestSlippyMapTiles.images, "image")
         self.assertEqual(len(dataset), 3)
 
     def test_getitem(self):
-        dataset = SlippyMapTiles(TestSlippyMapTiles.images)
+        dataset = SlippyMapTiles(TestSlippyMapTiles.images, "image")
         image, tile = dataset[0]
 
         assert tile == mercantile.Tile(69105, 105093, 18)
-        # Inspired by: https://github.com/python-pillow/Pillow/blob/master/Tests/test_image.py#L37-L38
-        self.assertEqual(repr(image)[:45], "<PIL.JpegImagePlugin.JpegImageFile image mode")
-        self.assertEqual(image.size, (512, 512))
+        self.assertEqual(image.size, 512 * 512 * 3)
+        self.assertEqual(image.shape, (512, 512, 3))
 
     def test_getitem_with_transform(self):
         # TODO

From fd4ca3b79cadcbf36668704abe5cad1dde414e83 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 22 Oct 2018 17:34:04 +0200
Subject: [PATCH 10/97] add tile tools

---
 deps/requirements-lock.txt |  1 +
 deps/requirements.txt      |  1 +
 robosat/tools/__main__.py  |  2 +
 robosat/tools/tile.py      | 81 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 85 insertions(+)
 create mode 100644 robosat/tools/tile.py

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 15b4ddad..eace5c68 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -42,3 +42,4 @@ torchvision==0.2.1
 tqdm==4.23.4
 urllib3==1.22
 Werkzeug==0.14.1
+rio_tiler==1.0a7
diff --git a/deps/requirements.txt b/deps/requirements.txt
index 423e11e7..30fe1d9d 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -17,3 +17,4 @@ rtree
 pyproj
 toml
 pytest
+rio_tiler
diff --git a/robosat/tools/__main__.py b/robosat/tools/__main__.py
index a4cf2f0b..ac63459a 100644
--- a/robosat/tools/__main__.py
+++ b/robosat/tools/__main__.py
@@ -16,6 +16,7 @@
     rasterize,
     serve,
     subset,
+    tile,
     train,
     weights,
 )
@@ -27,6 +28,7 @@ def add_parsers():
 
     # Add your tool's entry point below.
 
+    tile.add_parser(subparser)
     extract.add_parser(subparser)
     cover.add_parser(subparser)
     download.add_parser(subparser)
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
new file mode 100644
index 00000000..a0651309
--- /dev/null
+++ b/robosat/tools/tile.py
@@ -0,0 +1,81 @@
+import os
+import sys
+import argparse
+from tqdm import tqdm
+
+import numpy as np
+from PIL import Image
+
+import mercantile
+from rio_tiler import main as tiler
+from robosat.config import load_config
+from robosat.colors import make_palette
+
+
+def add_parser(subparser):
+    parser = subparser.add_parser(
+        "tile", help="tile a raster image", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument("raster", type=str, help="path to the raster to tile")
+    parser.add_argument("out", type=str, help="directory to write tiles")
+    parser.add_argument("--size", type=int, default=512, help="size of tiles side in pixels")
+    parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
+    parser.add_argument("--type", type=str, default="image", help="image or label tiling")
+    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, needed for label tiling")
+    parser.add_argument("--no_edges", type=bool, help="don't generate edges tiles")
+
+    parser.set_defaults(func=main)
+
+
+def main(args):
+
+    if args.type == "label":
+        try:
+            dataset = load_config(args.dataset)
+        except:
+            print("Error: Unable to load DataSet config file", file=sys.stderr)
+            sys.exit()
+
+        classes = dataset["common"]["classes"]
+        colors = dataset["common"]["colors"]
+        assert len(classes) == len(colors), "classes and colors coincide"
+        assert len(colors) == 2, "only binary models supported right now"
+
+    bounds = tiler.bounds(args.raster)["bounds"]
+    tiles = ([[x, y] for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])])
+
+    if args.no_edges:
+        edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
+        edges_y = (min(tiles, key=lambda xy: xy[1])[1]), (max(tiles, key=lambda xy: xy[1])[1])
+        tiles = [[x, y] for x, y in tiles if x not in edges_x and y not in edges_y]
+        assert len(tiles), "Error: Nothing left to tile, once remove the edges"
+
+    for x, y in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
+
+        os.makedirs(os.path.join(args.out, str(args.zoom), str(x)), exist_ok=True)
+        path = os.path.join(args.out, str(args.zoom), str(x), str(y))
+        data = tiler.tile(args.raster, x, y, args.zoom, tilesize=args.size)[0]
+
+        C, W, H = data.shape
+
+        if args.type == "label":
+            assert C == 1, "Error: Label raster input should be 1 band"
+
+            img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
+            img.putpalette(make_palette(colors[0], colors[1]))
+            img.save(path + ".png", optimize=True)
+
+        elif args.type == "image":
+            assert C == 1 or C == 3, "Error: Image raster input should be either 1 or 3 bands"
+
+            # GeoTiff could be 16 or 32bits
+            if data.dtype == "uint16":
+                 data = np.uint8(data / 256)
+            elif data.dtype == "uint32":
+                 data = np.uint8(data / (256 * 256))
+
+            if C == 1:
+                Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ".png", optimize=True)
+            elif C == 3:
+                Image.fromarray(data, mode="RGB").save(path + ".webp", optimize=True)

From 29935f66e82126243c4f2ed95c365e9c06a62b79 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 22 Oct 2018 17:51:11 +0200
Subject: [PATCH 11/97] black format

---
 robosat/tools/tile.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index a0651309..7ea571d2 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -43,7 +43,7 @@ def main(args):
         assert len(colors) == 2, "only binary models supported right now"
 
     bounds = tiler.bounds(args.raster)["bounds"]
-    tiles = ([[x, y] for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])])
+    tiles = [[x, y] for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])]
 
     if args.no_edges:
         edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
@@ -71,9 +71,9 @@ def main(args):
 
             # GeoTiff could be 16 or 32bits
             if data.dtype == "uint16":
-                 data = np.uint8(data / 256)
+                data = np.uint8(data / 256)
             elif data.dtype == "uint32":
-                 data = np.uint8(data / (256 * 256))
+                data = np.uint8(data / (256 * 256))
 
             if C == 1:
                 Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ".png", optimize=True)

From 2e97d16dc697b597805e50fd72dc45be5c0eceea Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 22 Oct 2018 18:47:08 +0200
Subject: [PATCH 12/97] refactor resize data augmentation

---
 config/model-unet.toml |  5 +----
 robosat/tools/train.py |  7 +++----
 robosat/transforms.py  | 25 ++++++++++++-------------
 3 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 44659055..0e325362 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -33,8 +33,5 @@
   # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
   loss = 'Lovasz'
 
-  # Resize factor to input image resolution
-  image_resize_factor = 1
-
   # Data augmentation, Flip or Rotate probabilty
-  flip_or_rotate_prob = 0.25
+  data_augmentation = 0.75
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 689bdd44..d55f16cd 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -119,8 +119,7 @@ def map_location(storage, _):
     log.log("--- Hyper Parameters on Dataset: {} ---".format(dataset["common"]["dataset"]))
     log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
     log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
-    log.log("Image Resize factor:\t {}".format(model["opt"]["image_resize_factor"]))
-    log.log("Flip or Rotate prob:\t {}".format(model["opt"]["flip_or_rotate_prob"]))
+    log.log("Data Augmentation:\t {}".format(model["opt"]["data_augmentation"]))
     log.log("Learning Rate:\t\t {}".format(model["opt"]["lr"]))
     log.log("Weight Decay:\t\t {}".format(model["opt"]["decay"]))
     log.log("Loss function:\t\t {}".format(model["opt"]["loss"]))
@@ -254,8 +253,8 @@ def get_dataset_loaders(model, dataset, workers):
 
     transform = JointCompose(
         [
-            JointResize(model["opt"]["image_resize_factor"]),
-            JointRandomFlipOrRotate(model["opt"]["flip_or_rotate_prob"]),
+            JointResize(model["common"]["image_size"]),
+            JointRandomFlipOrRotate(model["opt"]["data_augmentation"]),
             JointTransform(ImageToTensor(), MaskToTensor()),
             JointTransform(Normalize(mean=mean, std=std), None),
         ]
diff --git a/robosat/transforms.py b/robosat/transforms.py
index 3c18c982..2df5ee20 100644
--- a/robosat/transforms.py
+++ b/robosat/transforms.py
@@ -156,14 +156,13 @@ class JointResize:
     """Callable to resize image and its mask
     """
 
-    def __init__(self, f):
+    def __init__(self, size):
         """Creates an `JointResize` instance.
 
         Args:
-          f: the desired resize factor
+          size: the desired square side size
         """
-        assert not (f % 2) or not (int(1 / f) % 2) or f == 1, "Invalid resize factor value"
-        self.f = f
+        self.hw = (size, size)
 
     def __call__(self, image, mask):
         """Resize image and its mask
@@ -176,14 +175,14 @@ def __call__(self, image, mask):
           The (image, mask) tuple resized
         """
 
-        if self.f == 1:
-            return image, mask
-        elif self.f > 1:
-            image_interpolation = cv2.INTER_AREA
+        if self.hw == image.shape[0:2]:
+            pass
+        elif self.hw[0] < image.shape[0] and self.hw[1] < image.shape[1]:
+            image = cv2.resize(image, self.hw, interpolation=cv2.INTER_AREA)
         else:
-            image_interpolation = cv2.INTER_LINEAR
+            image = cv2.resize(image, self.hw, interpolation=cv2.INTER_LINEAR)
+
+        if self.hw != mask.shape:
+            mask = cv2.resize(mask, self.hw, interpolation=cv2.INTER_NEAREST)
 
-        return (
-            cv2.resize(image, None, fx=self.f, fy=self.f, interpolation=image_interpolation),
-            cv2.resize(mask, None, fx=self.f, fy=self.f, interpolation=cv2.INTER_NEAREST),
-        )
+        return image, mask

From dd15457dbe4fdcd7c9a4c74b8e9890d3999a1b3a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 22 Oct 2018 19:01:28 +0200
Subject: [PATCH 13/97] typo

---
 config/model-unet.toml | 2 +-
 robosat/tools/tile.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 0e325362..9724c208 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -33,5 +33,5 @@
   # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
   loss = 'Lovasz'
 
-  # Data augmentation, Flip or Rotate probabilty
+  # Data augmentation, Flip or Rotate probability
   data_augmentation = 0.75
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 7ea571d2..12677523 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -14,7 +14,7 @@
 
 def add_parser(subparser):
     parser = subparser.add_parser(
-        "tile", help="tile a raster image", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        "tile", help="tile a raster image or label", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
     parser.add_argument("raster", type=str, help="path to the raster to tile")

From 822bae715d635b9c53691a46429c85071f39729a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 22 Oct 2018 21:36:32 +0200
Subject: [PATCH 14/97] Fix issue relative to OpenCV switch.

---
 robosat/tools/serve.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tools/serve.py b/robosat/tools/serve.py
index 2bc391f4..28aaa651 100644
--- a/robosat/tools/serve.py
+++ b/robosat/tools/serve.py
@@ -63,7 +63,7 @@ def tile(z, x, y):
     if not res:
         abort(500)
 
-    image = cv2.cvtColor(cv2.imread(res), cv2.COLOR_BGR2RGB)
+    image = cv2.imdecode(np.asarray(bytearray(res.read()), dtype=np.uint8), cv2.COLOR_BGR2RGB)
 
     mask = predictor.segment(image)
 

From 0b86808d01d96dffaadbf4105e027fd9c7790e9a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 23 Oct 2018 12:50:05 +0200
Subject: [PATCH 15/97] Fix axes order on labels tiles. Add label_thresold
 option. Few clean stuff

---
 robosat/tools/tile.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 12677523..16674a2a 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -23,7 +23,8 @@ def add_parser(subparser):
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, needed for label tiling")
-    parser.add_argument("--no_edges", type=bool, help="don't generate edges tiles")
+    parser.add_argument("--no_edges", action="store_true", help="skip to generate edges tiles")
+    parser.add_argument("--label_thresold", type=int, default=1, help="label value thresold")
 
     parser.set_defaults(func=main)
 
@@ -49,7 +50,7 @@ def main(args):
         edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
         edges_y = (min(tiles, key=lambda xy: xy[1])[1]), (max(tiles, key=lambda xy: xy[1])[1])
         tiles = [[x, y] for x, y in tiles if x not in edges_x and y not in edges_y]
-        assert len(tiles), "Error: Nothing left to tile, once remove the edges"
+        assert len(tiles), "Error: Nothing left to tile, once the edges removed"
 
     for x, y in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
 
@@ -62,6 +63,9 @@ def main(args):
         if args.type == "label":
             assert C == 1, "Error: Label raster input should be 1 band"
 
+            data[data < args.label_thresold] = 0
+            data[data >= args.label_thresold] = 1
+
             img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
             img.putpalette(make_palette(colors[0], colors[1]))
             img.save(path + ".png", optimize=True)
@@ -78,4 +82,8 @@ def main(args):
             if C == 1:
                 Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ".png", optimize=True)
             elif C == 3:
-                Image.fromarray(data, mode="RGB").save(path + ".webp", optimize=True)
+                Image.fromarray(np.swapaxes(data, 0, 2), mode="RGB").save(path + ".webp", optimize=True)
+
+        else:
+            print("Error: Unknown type, should be either 'image' or 'label'", file=sys.stderr)
+            sys.exit()

From 1e4363200643c2b1c07ff25e71ae25491f37bc8c Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 26 Oct 2018 19:07:13 +0200
Subject: [PATCH 16/97] Add masks_ouput option in predict tool

---
 robosat/tools/predict.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 88edcc50..1a705d71 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -16,7 +16,7 @@
 from robosat.datasets import BufferedSlippyMapDirectory
 from robosat.unet import UNet
 from robosat.config import load_config
-from robosat.colors import continuous_palette_for_color
+from robosat.colors import continuous_palette_for_color, make_palette
 from robosat.transforms import ImageToTensor
 
 
@@ -36,6 +36,7 @@ def add_parser(subparser):
     parser.add_argument("probs", type=str, help="directory to save slippy map probability masks to")
     parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
 
     parser.set_defaults(func=main)
 
@@ -97,12 +98,15 @@ def map_location(storage, _):
                 assert np.allclose(np.sum(prob, axis=0), 1.), "single channel requires probabilities to sum up to one"
                 foreground = prob[1:, :, :]
 
-                anchors = np.linspace(0, 1, 256)
-                quantized = np.digitize(foreground, anchors).astype(np.uint8)
+                if args.masks_output:
+                    image = np.around(foreground)
+                    palette = make_palette("denim", "orange")
 
-                palette = continuous_palette_for_color("pink", 256)
+                else:
+                    image = np.digitize(foreground, np.linspace(0, 1, 256))
+                    palette = continuous_palette_for_color("pink", 256)
 
-                out = Image.fromarray(quantized.squeeze(), mode="P")
+                out = Image.fromarray(image.squeeze().astype(np.uint8), mode="P")
                 out.putpalette(palette)
 
                 os.makedirs(os.path.join(args.probs, str(z), str(x)), exist_ok=True)

From 48115d2a5bef600c2c7b1df4a3a43dfc0a23a4ce Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 26 Oct 2018 19:40:40 +0200
Subject: [PATCH 17/97] Update comments (related to output_masks)

---
 robosat/tools/predict.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 1a705d71..67406cd2 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -91,18 +91,18 @@ def map_location(storage, _):
                 # we predicted on buffered tiles; now get back probs for original image
                 prob = directory.unbuffer(prob)
 
-                # Quantize the floating point probabilities in [0,1] to [0,255] and store
-                # a single-channel `.png` file with a continuous color palette attached.
-
                 assert prob.shape[0] == 2, "single channel requires binary model"
                 assert np.allclose(np.sum(prob, axis=0), 1.), "single channel requires probabilities to sum up to one"
+
                 foreground = prob[1:, :, :]
 
                 if args.masks_output:
+                    # Quantize the floating point prob in [0,1] to {0,1} with a fixed palette attached
                     image = np.around(foreground)
                     palette = make_palette("denim", "orange")
 
                 else:
+                    # Quantize the floating point prob in [0,1] to [0,255] with a continuous color palette attached
                     image = np.digitize(foreground, np.linspace(0, 1, 256))
                     palette = continuous_palette_for_color("pink", 256)
 

From 82c2bcd867d8bfc3f36db8b32ac67fa9db4b48bb Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 27 Oct 2018 11:52:44 +0200
Subject: [PATCH 18/97] in train, resume option don't need an explicit yes.
 User Friendly

---
 robosat/tools/train.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index d55f16cd..32c1bf0a 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -46,7 +46,7 @@ def add_parser(subparser):
     parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--checkpoint", type=str, required=False, help="path to a model checkpoint (to retrain)")
-    parser.add_argument("--resume", type=bool, default=False, help="resume training or fine-tuning (if checkpoint)")
+    parser.add_argument("--resume", action="store_true", help="resume training (imply to provide a checkpoint)")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
 
     parser.set_defaults(func=main)

From f2e640586b5be6cab7a3236f01883621abcc1d61 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 27 Oct 2018 13:02:12 +0200
Subject: [PATCH 19/97] Add leaflet ouput option

---
 robosat/tools/compare.py         |  4 ++++
 robosat/tools/download.py        |  5 +++++
 robosat/tools/masks.py           |  6 ++++++
 robosat/tools/predict.py         |  5 +++++
 robosat/tools/rasterize.py       |  5 +++++
 robosat/tools/subset.py          | 11 ++++++++---
 robosat/tools/templates/map.html |  6 +++---
 robosat/tools/tile.py            | 14 +++++++++++---
 robosat/utils.py                 | 20 ++++++++++++++++++++
 9 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 956fc969..c50cffca 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -20,6 +20,7 @@ def add_parser(subparser):
     parser.add_argument("masks", type=str, nargs="+", help="slippy map directories to read masks from")
     parser.add_argument("--minimum", type=float, default=0.0, help="minimum percentage of mask not background")
     parser.add_argument("--maximum", type=float, default=1.0, help="maximum percentage of mask not background")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -65,3 +66,6 @@ def main(args):
         os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
         path = os.path.join(args.out, z, x, "{}.png".format(y))
         combined.save(path, optimize=True)
+
+    if args.leaflet:
+        leaflet(args.out, args.leaflet, images, ".png")
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 1be70ff2..30f1e47c 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -9,6 +9,7 @@
 from tqdm import tqdm
 
 from robosat.tiles import tiles_from_csv, fetch_image, tile_to_bbox
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -25,6 +26,7 @@ def add_parser(subparser):
     parser.add_argument("--timeout", type=int, default=10, help="server request timeout (in seconds)")
     parser.add_argument("tiles", type=str, help="path to .csv tiles file")
     parser.add_argument("out", type=str, help="path to slippy map directory for storing tiles")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -87,3 +89,6 @@ def worker(tile):
             for tile, url, ok in executor.map(worker, tiles):
                 if not ok:
                     print("Warning:\n {} failed, skipping.\n {}\n".format(tile, url), file=sys.stderr)
+
+    if args.leaflet:
+        leaflet(args.out, args.leaflet, tiles, args.ext)
diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index 310956c0..d1fb4b1e 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -9,6 +9,7 @@
 
 from robosat.tiles import tiles_from_slippy_map
 from robosat.colors import make_palette
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -21,6 +22,7 @@ def add_parser(subparser):
     parser.add_argument("masks", type=str, help="slippy map directory to save masks to")
     parser.add_argument("probs", type=str, nargs="+", help="slippy map directories with class probabilities")
     parser.add_argument("--weights", type=float, nargs="+", help="weights for weighted average soft-voting")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -68,6 +70,10 @@ def load(path):
         path = os.path.join(args.masks, str(z), str(x), str(y) + ".png")
         out.save(path, optimize=True)
 
+    if args.leaflet:
+        tiles = [tile for tile, _ in list(tiles_from_slippy_map(args.probs[0]))]
+        leaflet(args.masks, args.leaflet, tiles, ".png")
+
 
 def softvote(probs, axis=0, weights=None):
     """Weighted average soft-voting to transform class probabilities into class indices.
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 67406cd2..3a0b0591 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -18,6 +18,7 @@
 from robosat.config import load_config
 from robosat.colors import continuous_palette_for_color, make_palette
 from robosat.transforms import ImageToTensor
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -37,6 +38,7 @@ def add_parser(subparser):
     parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -113,3 +115,6 @@ def map_location(storage, _):
                 path = os.path.join(args.probs, str(z), str(x), str(y) + ".png")
 
                 out.save(path, optimize=True)
+
+        if args.leaflet:
+            leaflet(args.probs, args.leaflet, tiles, ".png")
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 30ce1c01..adfa3b40 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -19,6 +19,7 @@
 from robosat.config import load_config
 from robosat.colors import make_palette
 from robosat.tiles import tiles_from_csv
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -32,6 +33,7 @@ def add_parser(subparser):
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--size", type=int, default=512, help="size of rasterized image tiles in pixels")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -123,3 +125,6 @@ def main(args):
         os.makedirs(out_path, exist_ok=True)
 
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
+
+    if args.leaflet:
+        leaflet(args.out, args.leaflet, args.tiles, ".png")
diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index 14f0a70a..db2ac466 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -5,6 +5,7 @@
 from tqdm import tqdm
 
 from robosat.tiles import tiles_from_slippy_map, tiles_from_csv
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -16,6 +17,7 @@ def add_parser(subparser):
     parser.add_argument("images", type=str, help="directory to read slippy map image tiles from for filtering")
     parser.add_argument("tiles", type=str, help="csv to filter images by")
     parser.add_argument("out", type=str, help="directory to save filtered images to")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -29,10 +31,13 @@ def main(args):
         if tile not in tiles:
             continue
 
-        # The extention also includes the period.
-        extention = os.path.splitext(src)[1]
+        # The extension also includes the period.
+        extension = os.path.splitext(src)[1]
 
         os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
-        dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}{}".format(tile.y, extention))
+        dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}{}".format(tile.y, extension))
 
         shutil.copyfile(src, dst)
+
+    if args.leaflet:
+        leaflet(args.out, args.leaflet, tiles, extension)
diff --git a/robosat/tools/templates/map.html b/robosat/tools/templates/map.html
index 18f6b7a3..d16bc5d7 100644
--- a/robosat/tools/templates/map.html
+++ b/robosat/tools/templates/map.html
@@ -45,7 +45,7 @@
 
 var beforeMap = new mapboxgl.Map({
   container: 'before',
-  center: [-84.37, 33.75],
+  center: [45.5, 5.5],
   zoom: 18,
   minZoom: 5,
   maxZoom: 20,
@@ -55,7 +55,7 @@
 
 var afterMap = new mapboxgl.Map({
   container: 'after',
-  center: [-84.37, 33.75],
+  center: [45.5, 5.5],
   zoom: 18,
   minZoom: 5,
   maxZoom: 20,
@@ -71,7 +71,7 @@
     'type': 'raster',
     'source': {
       'type': 'raster',
-      'tiles': [ 'http://127.0.0.1:5000/{z}/{x}/{y}.png' ],
+      'tiles': [ 'http://127.0.0.1/rs/gl/validation/probs/{z}/{x}/{y}.png' ],
       'tileSize': {{ size }}
     }
   });
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 16674a2a..69e3822b 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -10,6 +10,7 @@
 from rio_tiler import main as tiler
 from robosat.config import load_config
 from robosat.colors import make_palette
+from robosat.utils import leaflet
 
 
 def add_parser(subparser):
@@ -25,6 +26,7 @@ def add_parser(subparser):
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, needed for label tiling")
     parser.add_argument("--no_edges", action="store_true", help="skip to generate edges tiles")
     parser.add_argument("--label_thresold", type=int, default=1, help="label value thresold")
+    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -66,9 +68,10 @@ def main(args):
             data[data < args.label_thresold] = 0
             data[data >= args.label_thresold] = 1
 
+            ext = ".png"
             img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
             img.putpalette(make_palette(colors[0], colors[1]))
-            img.save(path + ".png", optimize=True)
+            img.save(path + ext, optimize=True)
 
         elif args.type == "image":
             assert C == 1 or C == 3, "Error: Image raster input should be either 1 or 3 bands"
@@ -80,10 +83,15 @@ def main(args):
                 data = np.uint8(data / (256 * 256))
 
             if C == 1:
-                Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ".png", optimize=True)
+                ext = ".png"
+                Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ext, optimize=True)
             elif C == 3:
-                Image.fromarray(np.swapaxes(data, 0, 2), mode="RGB").save(path + ".webp", optimize=True)
+                ext = ".webp"
+                Image.fromarray(np.swapaxes(data, 0, 2), mode="RGB").save(path + ext, optimize=True)
 
         else:
             print("Error: Unknown type, should be either 'image' or 'label'", file=sys.stderr)
             sys.exit()
+
+    if args.leaflet:
+        leaflet(args.out, args.leaflet, tiles, ext)
diff --git a/robosat/utils.py b/robosat/utils.py
index 60028221..28400169 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -1,3 +1,6 @@
+import re
+import os 
+from robosat.tiles import pixel_to_location
 import matplotlib
 
 matplotlib.use("Agg")
@@ -20,3 +23,20 @@ def plot(out, history):
 
     plt.savefig(out, format="png")
     plt.close()
+
+
+def leaflet(out, base_url, tiles, ext):
+
+    leaflet = open("./robosat/tools/templates/leaflet.html", "r").read()
+    leaflet = re.sub("{{base_url}}", base_url, leaflet)
+    leaflet = re.sub("{{ext}}", ext, leaflet)
+
+    # Could surely be improve, but for now, took the first tile to center on
+    tile = (list(tiles)[0])
+    x, y, z = map(int, [tile.x, tile.y, tile.z])
+    leaflet = re.sub("{{zoom}}", str(z), leaflet)
+    leaflet = re.sub("{{center}}", str(list(pixel_to_location(tile, 0.5, 0.5))[::-1]), leaflet)
+
+    f = open(os.path.join(out, "index.html"), "w", encoding="utf-8")
+    f.write(leaflet)
+    f.close()

From b0c554c29a928af1134e977126056b86cd66e6ff Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 27 Oct 2018 13:09:16 +0200
Subject: [PATCH 20/97] reverse map templates changes. Add leaflet html
 template

---
 robosat/tools/templates/leaflet.html | 21 +++++++++++++++++++++
 robosat/tools/templates/map.html     |  6 +++---
 2 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 robosat/tools/templates/leaflet.html

diff --git a/robosat/tools/templates/leaflet.html b/robosat/tools/templates/leaflet.html
new file mode 100644
index 00000000..556ab1ad
--- /dev/null
+++ b/robosat/tools/templates/leaflet.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>RoboSat LeafLet Client</title>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.3.4/dist/leaflet.css"
+        integrity="sha512-puBpdR0798OZvTTbP4A8Ix/l+A4dHDD0DGqYW6RQ+9jxkRFclaxxQb/SJAWZfWAkuyeQUytO7+7N4QKrDh+drA=="
+	crossorigin="" />
+  <script src="https://unpkg.com/leaflet@1.3.4/dist/leaflet.js"
+	  integrity="sha512-nMMmRyTVoLYqjP9hrbed9S+FzjZHW5gY1TWCHA5ckwXZBadntCNs8kEqAWdrb9O7rxbCaA4lKTIWjDXZxflOcA=="
+	  crossorigin=""></script>
+</head>
+<body>
+  <div id="mapid" style="width:100%; height:100vh;"></div>
+  <script>
+    var m = L.map("mapid", {zoomControl: false, boxZoom: false, scrollWheelZoom: false}).setView({{center}}, {{zoom}});
+    L.tileLayer("{{base_url}}/{z}/{x}/{y}{{ext}}").addTo(m);
+  </script>
+</body>
+</html>
diff --git a/robosat/tools/templates/map.html b/robosat/tools/templates/map.html
index d16bc5d7..18f6b7a3 100644
--- a/robosat/tools/templates/map.html
+++ b/robosat/tools/templates/map.html
@@ -45,7 +45,7 @@
 
 var beforeMap = new mapboxgl.Map({
   container: 'before',
-  center: [45.5, 5.5],
+  center: [-84.37, 33.75],
   zoom: 18,
   minZoom: 5,
   maxZoom: 20,
@@ -55,7 +55,7 @@
 
 var afterMap = new mapboxgl.Map({
   container: 'after',
-  center: [45.5, 5.5],
+  center: [-84.37, 33.75],
   zoom: 18,
   minZoom: 5,
   maxZoom: 20,
@@ -71,7 +71,7 @@
     'type': 'raster',
     'source': {
       'type': 'raster',
-      'tiles': [ 'http://127.0.0.1/rs/gl/validation/probs/{z}/{x}/{y}.png' ],
+      'tiles': [ 'http://127.0.0.1:5000/{z}/{x}/{y}.png' ],
       'tileSize': {{ size }}
     }
   });

From fab73f9771deea46d4a4dd547d7173827e7ff6b8 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 18:28:15 +0100
Subject: [PATCH 21/97] Move from pillow to pillow-simd. Significant perf
 improvement. Did'nt test Docker instances

---
 deps/requirements-lock.txt | 2 +-
 deps/requirements.txt      | 2 +-
 docker/Dockerfile.cpu      | 2 +-
 docker/Dockerfile.gpu      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index eace5c68..2a467791 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -20,7 +20,7 @@ more-itertools==4.2.0
 numpy==1.14.4
 opencv-contrib-python==3.4.1.15
 osmium==2.14.1
-Pillow==5.1.0
+Pillow-simd==5.3.0.post0
 pluggy==0.6.0
 py==1.5.3
 pyparsing==2.2.0
diff --git a/deps/requirements.txt b/deps/requirements.txt
index 30fe1d9d..7a8b1bfb 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -1,6 +1,6 @@
 torchvision
 numpy
-pillow
+pillow-simd
 scipy
 opencv-contrib-python
 tqdm
diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu
index d824fd5f..01237605 100644
--- a/docker/Dockerfile.cpu
+++ b/docker/Dockerfile.cpu
@@ -6,7 +6,7 @@ FROM ubuntu:16.04
 # See: https://github.com/skvark/opencv-python/issues/90
 RUN apt-get update -qq && \
     apt-get install -qq -y -o quiet=1 \
-    python3 python3-dev python3-tk python3-pip build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev libsm6
+    python3 python3-dev python3-tk python3-pip build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev libsm6 libwebp-dev libjpeg-turbo8-dev
 
 WORKDIR /app
 ADD . /app
diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu
index 776ef8b3..02b3baa5 100644
--- a/docker/Dockerfile.gpu
+++ b/docker/Dockerfile.gpu
@@ -6,7 +6,7 @@ FROM nvidia/cuda:9.1-cudnn7-runtime-ubuntu16.04
 # See: https://github.com/skvark/opencv-python/issues/90
 RUN apt-get update -qq && \
     apt-get install -qq -y -o quiet=1 \
-    python3 python3-dev python3-tk python3-pip build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev libsm6
+    python3 python3-dev python3-tk python3-pip build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev libsm6 libwebp-dev libjpeg-turbo8-dev
 
 WORKDIR /app
 ADD . /app

From e68d8f5274ef60ccad233c1f43ad658a8dfdb530 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 18:30:25 +0100
Subject: [PATCH 22/97] Variable name homogenize

---
 robosat/transforms.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/robosat/transforms.py b/robosat/transforms.py
index 2df5ee20..cbb0d0f3 100644
--- a/robosat/transforms.py
+++ b/robosat/transforms.py
@@ -28,17 +28,17 @@ class MaskToTensor:
     """Callable to convert an OpenCV H,W image into a PyTorch tensor.
     """
 
-    def __call__(self, tensor):
-        """Converts the image into a tensor.
+    def __call__(self, mask):
+        """Converts the mask into a tensor.
 
         Args:
-          image: the image to convert into a PyTorch tensor.
+          mask: the mask to convert into a PyTorch tensor.
 
         Returns:
           The converted PyTorch tensor.
         """
 
-        return torch.from_numpy(tensor).long()
+        return torch.from_numpy(mask).long()
 
 
 class JointCompose:

From 236f471c1005590394a81b98249b3167ba97e363 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 18:32:27 +0100
Subject: [PATCH 23/97] Add robosat log handling. User friendly log output if
 some tiles was already downloaded

---
 robosat/tools/download.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 30f1e47c..ad3b068d 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -10,6 +10,7 @@
 
 from robosat.tiles import tiles_from_csv, fetch_image, tile_to_bbox
 from robosat.utils import leaflet
+from robosat.log import Log
 
 
 def add_parser(subparser):
@@ -33,10 +34,15 @@ def add_parser(subparser):
 
 def main(args):
     tiles = list(tiles_from_csv(args.tiles))
+    already_dl = 0
 
     with requests.Session() as session:
         num_workers = args.rate
 
+        os.makedirs(os.path.join(args.out), exist_ok=True)
+        log = Log(os.path.join(args.out, "log"), out=sys.stderr)
+        log.log("Begin download from {}".format(args.url))
+
         # tqdm has problems with concurrent.futures.ThreadPoolExecutor; explicitly call `.update`
         # https://github.com/tqdm/tqdm/issues/97
         progress = tqdm(total=len(tiles), ascii=True, unit="image")
@@ -87,8 +93,13 @@ def worker(tile):
                 return tile, url, True
 
             for tile, url, ok in executor.map(worker, tiles):
+                if not url and ok:
+                    already_dl += 1
                 if not ok:
-                    print("Warning:\n {} failed, skipping.\n {}\n".format(tile, url), file=sys.stderr)
+                    log.log("Warning:\n {} failed, skipping.\n {}\n".format(tile, url))
+
+    if already_dl:
+        log.log("Notice:\n {} tiles already downloads previously, so skipped now.".format(already_dl))
 
     if args.leaflet:
         leaflet(args.out, args.leaflet, tiles, args.ext)

From 69ef55708e55520bae390bf1130169816074bb6a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 18:33:40 +0100
Subject: [PATCH 24/97] Add bbox option to cover tool

---
 robosat/tools/cover.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/robosat/tools/cover.py b/robosat/tools/cover.py
index ee5a2931..213ae346 100644
--- a/robosat/tools/cover.py
+++ b/robosat/tools/cover.py
@@ -1,37 +1,49 @@
 import argparse
 import csv
 import json
+import sys
 
 from supermercado import burntiles
+from mercantile import tiles
 from tqdm import tqdm
 
 
 def add_parser(subparser):
     parser = subparser.add_parser(
         "cover",
-        help="generates tiles covering GeoJSON features",
+        help="generates tiles covering GeoJSON features or lat/lon Bbox",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
-    parser.add_argument("features", type=str, help="path to GeoJSON features")
+    parser.add_argument("--features", type=str, help="path to GeoJSON features")
+    parser.add_argument("--bbox", type=str, help="bbox expressed in lat/lon (i.e EPSG:4326)")
     parser.add_argument("out", type=str, help="path to csv file to store tiles in")
 
     parser.set_defaults(func=main)
 
 
 def main(args):
-    with open(args.features) as f:
-        features = json.load(f)
 
-    tiles = []
+    cover = []
 
-    for feature in tqdm(features["features"], ascii=True, unit="feature"):
-        tiles.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist()))
+    if args.features:
+        with open(args.features) as f:
+            features = json.load(f)
 
-    # tiles can overlap for multiple features; unique tile ids
-    tiles = list(set(tiles))
+        for feature in tqdm(features["features"], ascii=True, unit="feature"):
+            cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist()))
+
+        # tiles can overlap for multiple features; unique tile ids
+        cover = list(set(cover))
+
+    elif args.bbox:
+        west, south, east, north = map(float, args.bbox.split(","))
+        cover = tiles(west, south, east, north, args.zoom)
+
+    else:
+        sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox")
 
     with open(args.out, "w") as fp:
         writer = csv.writer(fp)
-        writer.writerows(tiles)
+        writer.writerows(cover)

From 249107bbec67cf53debd004ab91cee4ada0244ad Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 19:25:16 +0100
Subject: [PATCH 25/97] remove period from extension

---
 robosat/tools/masks.py               | 2 +-
 robosat/tools/predict.py             | 2 +-
 robosat/tools/rasterize.py           | 2 +-
 robosat/tools/subset.py              | 5 ++---
 robosat/tools/templates/leaflet.html | 2 +-
 robosat/tools/tile.py                | 6 +++---
 6 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index d1fb4b1e..7d72501a 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -72,7 +72,7 @@ def load(path):
 
     if args.leaflet:
         tiles = [tile for tile, _ in list(tiles_from_slippy_map(args.probs[0]))]
-        leaflet(args.masks, args.leaflet, tiles, ".png")
+        leaflet(args.masks, args.leaflet, tiles, "png")
 
 
 def softvote(probs, axis=0, weights=None):
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 3a0b0591..16b6e3fd 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -117,4 +117,4 @@ def map_location(storage, _):
                 out.save(path, optimize=True)
 
         if args.leaflet:
-            leaflet(args.probs, args.leaflet, tiles, ".png")
+            leaflet(args.probs, args.leaflet, tiles, "png")
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index adfa3b40..7f2482f2 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -127,4 +127,4 @@ def main(args):
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, args.tiles, ".png")
+        leaflet(args.out, args.leaflet, tiles_from_csv(args.tiles), "png")
diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index db2ac466..a4d44ed9 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -31,11 +31,10 @@ def main(args):
         if tile not in tiles:
             continue
 
-        # The extension also includes the period.
-        extension = os.path.splitext(src)[1]
+        extension = os.path.splitext(src)[1][1:]
 
         os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
-        dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}{}".format(tile.y, extension))
+        dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension))
 
         shutil.copyfile(src, dst)
 
diff --git a/robosat/tools/templates/leaflet.html b/robosat/tools/templates/leaflet.html
index 556ab1ad..6f22ab2e 100644
--- a/robosat/tools/templates/leaflet.html
+++ b/robosat/tools/templates/leaflet.html
@@ -15,7 +15,7 @@
   <div id="mapid" style="width:100%; height:100vh;"></div>
   <script>
     var m = L.map("mapid", {zoomControl: false, boxZoom: false, scrollWheelZoom: false}).setView({{center}}, {{zoom}});
-    L.tileLayer("{{base_url}}/{z}/{x}/{y}{{ext}}").addTo(m);
+    L.tileLayer("{{base_url}}/{z}/{x}/{y}.{{ext}}").addTo(m);
   </script>
 </body>
 </html>
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 69e3822b..084a441c 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -68,7 +68,7 @@ def main(args):
             data[data < args.label_thresold] = 0
             data[data >= args.label_thresold] = 1
 
-            ext = ".png"
+            ext = "png"
             img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
             img.putpalette(make_palette(colors[0], colors[1]))
             img.save(path + ext, optimize=True)
@@ -83,10 +83,10 @@ def main(args):
                 data = np.uint8(data / (256 * 256))
 
             if C == 1:
-                ext = ".png"
+                ext = "png"
                 Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ext, optimize=True)
             elif C == 3:
-                ext = ".webp"
+                ext = "webp"
                 Image.fromarray(np.swapaxes(data, 0, 2), mode="RGB").save(path + ext, optimize=True)
 
         else:

From 14077300ed0e77dfbff503bc7e44cb64640e3c8b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 28 Oct 2018 22:46:38 +0100
Subject: [PATCH 26/97] Tile tool: few debug fixes and cleanup

---
 robosat/tools/tile.py | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 084a441c..21e8fda1 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -23,9 +23,9 @@ def add_parser(subparser):
     parser.add_argument("--size", type=int, default=512, help="size of tiles side in pixels")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, default="image", help="image or label tiling")
-    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, needed for label tiling")
+    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
     parser.add_argument("--no_edges", action="store_true", help="skip to generate edges tiles")
-    parser.add_argument("--label_thresold", type=int, default=1, help="label value thresold")
+    parser.add_argument("--label_threshold", type=int, default=1, help="label value threshold")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
@@ -37,8 +37,7 @@ def main(args):
         try:
             dataset = load_config(args.dataset)
         except:
-            print("Error: Unable to load DataSet config file", file=sys.stderr)
-            sys.exit()
+            sys.exit("Error: Unable to load DataSet config file")
 
         classes = dataset["common"]["classes"]
         colors = dataset["common"]["colors"]
@@ -46,32 +45,32 @@ def main(args):
         assert len(colors) == 2, "only binary models supported right now"
 
     bounds = tiler.bounds(args.raster)["bounds"]
-    tiles = [[x, y] for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])]
+    tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])]
 
     if args.no_edges:
         edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
         edges_y = (min(tiles, key=lambda xy: xy[1])[1]), (max(tiles, key=lambda xy: xy[1])[1])
-        tiles = [[x, y] for x, y in tiles if x not in edges_x and y not in edges_y]
+        tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in tiles if x not in edges_x and y not in edges_y]
         assert len(tiles), "Error: Nothing left to tile, once the edges removed"
 
-    for x, y in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
+    for tile in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
 
-        os.makedirs(os.path.join(args.out, str(args.zoom), str(x)), exist_ok=True)
-        path = os.path.join(args.out, str(args.zoom), str(x), str(y))
-        data = tiler.tile(args.raster, x, y, args.zoom, tilesize=args.size)[0]
+        os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
+        path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
+        data = tiler.tile(args.raster, tile.x, tile.y, args.zoom, tilesize=args.size)[0]
 
         C, W, H = data.shape
 
         if args.type == "label":
             assert C == 1, "Error: Label raster input should be 1 band"
 
-            data[data < args.label_thresold] = 0
-            data[data >= args.label_thresold] = 1
+            data[data < args.label_threshold] = 0
+            data[data >= args.label_threshold] = 1
 
             ext = "png"
             img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
             img.putpalette(make_palette(colors[0], colors[1]))
-            img.save(path + ext, optimize=True)
+            img.save("{}.{}".format(path, ext), optimize=True)
 
         elif args.type == "image":
             assert C == 1 or C == 3, "Error: Image raster input should be either 1 or 3 bands"
@@ -84,14 +83,13 @@ def main(args):
 
             if C == 1:
                 ext = "png"
-                Image.fromarray(np.squeeze(data, axis=0), mode="L").save(path + ext, optimize=True)
+                Image.fromarray(np.squeeze(data, axis=0), mode="L").save("{}.{}".format(path, ext), optimize=True)
             elif C == 3:
                 ext = "webp"
-                Image.fromarray(np.swapaxes(data, 0, 2), mode="RGB").save(path + ext, optimize=True)
+                Image.fromarray(np.moveaxis(data, 0, 2), mode="RGB").save("{}.{}".format(path, ext), optimize=True)
 
         else:
-            print("Error: Unknown type, should be either 'image' or 'label'", file=sys.stderr)
-            sys.exit()
+            sys.exit("Error: Unknown type, should be either 'image' or 'label'")
 
     if args.leaflet:
         leaflet(args.out, args.leaflet, tiles, ext)

From 2fe386441ca559c40782404d431474db38ef3c4a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 01:29:14 +0100
Subject: [PATCH 27/97] add complemenraty_palette function

---
 robosat/colors.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/robosat/colors.py b/robosat/colors.py
index 50a91dd0..f0238fdb 100644
--- a/robosat/colors.py
+++ b/robosat/colors.py
@@ -93,3 +93,16 @@ def continuous_palette_for_color(color, bins=256):
     assert len(palette) // 3 == bins
 
     return palette
+
+
+def complementary_palette(palette):
+
+    comp_palette = []
+    colors = [palette[i:i+3] for i in range(0, len(palette), 3)]
+
+    for color in colors:
+        r, g, b = [v for v in color]
+        h, s, v = colorsys.rgb_to_hsv(r, g, b)
+        comp_palette.extend(map(int, colorsys.hsv_to_rgb((h + 0.5) % 1, s, v)))
+
+    return comp_palette

From 3f27682cb7bca1b256d2ccdd88a8618097e5fa88 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 01:30:40 +0100
Subject: [PATCH 28/97] allow to explicitly choose open file mode, in log

---
 robosat/log.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/robosat/log.py b/robosat/log.py
index 2e7f9ff1..e59e3b50 100644
--- a/robosat/log.py
+++ b/robosat/log.py
@@ -10,9 +10,9 @@ class Log:
     """Create a log instance on a log file
     """
 
-    def __init__(self, path, out=sys.stdout):
+    def __init__(self, path, out=sys.stdout, mode="a"):
         self.out = out
-        self.fp = open(path, "a")
+        self.fp = open(path, mode)
         assert self.fp, "Unable to open log file"
 
     """Log a new message to the opened log file, and optionnaly on stdout or stderr too

From 8da0e8bec2f39efae682c8a57841b8757a8b643a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 01:32:15 +0100
Subject: [PATCH 29/97] Use NaN rather than Inf on 0 div exception. Allow to
 use metric either with mask or prob

---
 robosat/metrics.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/robosat/metrics.py b/robosat/metrics.py
index 3b523adf..333de523 100644
--- a/robosat/metrics.py
+++ b/robosat/metrics.py
@@ -24,16 +24,19 @@ def __init__(self, labels):
         self.fp = 0
         self.tp = 0
 
-    def add(self, actual, predicted):
+    def add(self, label, predicted, is_prob=True):
         """Adds an observation to the tracker.
 
         Args:
-          actual: the ground truth labels.
-          predicted: the predicted labels.
+          label: the ground truth labels.
+          predicted: the predicted prob or mask.
+          is_prob: as predicted could be either a prob or a mask.
         """
 
-        masks = torch.argmax(predicted, 0)
-        confusion = masks.view(-1).float() / actual.view(-1).float()
+        if is_prob:
+            predicted = torch.argmax(predicted, 0)
+
+        confusion = predicted.view(-1).float() / label.view(-1).float()
 
         self.tn += torch.sum(torch.isnan(confusion)).item()
         self.fn += torch.sum(confusion == float("inf")).item()
@@ -50,7 +53,7 @@ def get_miou(self):
         try:
             miou = np.nanmean([self.tn / (self.tn + self.fn + self.fp), self.tp / (self.tp + self.fn + self.fp)])
         except ZeroDivisionError:
-            miou = float("Inf")
+            miou = float("NaN")
 
         return miou
 
@@ -64,7 +67,7 @@ def get_fg_iou(self):
         try:
             iou = self.tp / (self.tp + self.fn + self.fp)
         except ZeroDivisionError:
-            iou = float("Inf")
+            iou = float("NaN")
 
         return iou
 
@@ -80,7 +83,7 @@ def get_mcc(self):
                 (self.tp + self.fp) * (self.tp + self.fn) * (self.tn + self.fp) * (self.tn + self.fn)
             )
         except ZeroDivisionError:
-            mcc = float("Inf")
+            mcc = float("NaN")
 
         return mcc
 

From b2a76dd15a955a9c249f97b8b5e5b0975d784d78 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 01:33:05 +0100
Subject: [PATCH 30/97] Bugfix on leaflet tiles

---
 robosat/tools/predict.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 16b6e3fd..1c8bfc94 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -14,6 +14,7 @@
 from PIL import Image
 
 from robosat.datasets import BufferedSlippyMapDirectory
+from robosat.tiles import tiles_from_slippy_map
 from robosat.unet import UNet
 from robosat.config import load_config
 from robosat.colors import continuous_palette_for_color, make_palette
@@ -116,5 +117,5 @@ def map_location(storage, _):
 
                 out.save(path, optimize=True)
 
-        if args.leaflet:
-            leaflet(args.probs, args.leaflet, tiles, "png")
+    if args.leaflet:
+        leaflet(args.probs, args.leaflet, [tile for tile, _ in tiles_from_slippy_map(args.tiles)], "png")

From 4946f6f52120dd8d3c0eb7562d982316fb262353 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 01:33:59 +0100
Subject: [PATCH 31/97] Whole refactor of compare tool. Now with 3 modes: side,
 diff, list

---
 robosat/tools/compare.py | 156 +++++++++++++++++++++++++++++----------
 1 file changed, 115 insertions(+), 41 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index c50cffca..118f782e 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -1,71 +1,145 @@
 import os
+import sys
+import math
+import torch
 import argparse
 
 from PIL import Image
 from tqdm import tqdm
 import numpy as np
 
+from robosat.colors import make_palette, complementary_palette
 from robosat.tiles import tiles_from_slippy_map
+from robosat.config import load_config
+from robosat.metrics import Metrics
+from robosat.utils import leaflet
+from robosat.log import Log
 
 
 def add_parser(subparser):
     parser = subparser.add_parser(
-        "compare",
-        help="compare images, labels and masks side by side",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        "compare", help="compare images, labels and masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    parser.add_argument("out", type=str, help="directory to save visualizations to")
-    parser.add_argument("images", type=str, help="directory to read slippy map images from")
-    parser.add_argument("labels", type=str, help="directory to read slippy map labels from")
-    parser.add_argument("masks", type=str, nargs="+", help="slippy map directories to read masks from")
-    parser.add_argument("--minimum", type=float, default=0.0, help="minimum percentage of mask not background")
-    parser.add_argument("--maximum", type=float, default=1.0, help="maximum percentage of mask not background")
+    parser.add_argument("out", type=str, help="directory to save output to (or path in list mode)")
+    parser.add_argument("--images", type=str, help="directory to read slippy map images from")
+    parser.add_argument("--labels", type=str, help="directory to read slippy map labels from")
+    parser.add_argument("--masks", type=str, help="directory to read slippy map masks from")
+    parser.add_argument("--dirs", type=str, nargs="+", help="slippy map directories to compares (in side mode)")
+    parser.add_argument("--mode", type=str, default="side", help="compare mode (e.g side, diff or list)")
+    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, (for diff and list modes)")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip if foreground ratio below, [0-100]")
+    parser.add_argument("--minimum_qod", type=float, default=100.0, help="redshift tile if QoD below, [0-100]")
 
     parser.set_defaults(func=main)
 
 
+def compare(mask, label, classes):
+
+    # TODO: Still binary class centric
+
+    metrics = Metrics(classes)
+    metrics.add(torch.from_numpy(label), torch.from_numpy(mask), is_prob=False)
+    fg_iou = metrics.get_fg_iou()
+
+    fg_ratio = 100 * max(np.sum(mask != 0), np.sum(label != 0)) / mask.size
+    dist = 0.0 if math.isnan(fg_iou) else 1.0 - fg_iou
+
+    qod = 100 - (dist * (math.log(fg_ratio + 1.0) + np.finfo(float).eps) * (100 / math.log(100)))
+    qod = 0.0 if qod < 0.0 else qod  # Corner case prophilaxy
+
+    return dist, fg_ratio, qod
+
+
 def main(args):
-    images = tiles_from_slippy_map(args.images)
 
-    for tile, path in tqdm(list(images), desc="Compare", unit="image", ascii=True):
-        x, y, z = list(map(str, tile))
+    if args.mode == "side":
+        if not args.dirs or len(args.dirs) < 2:
+            sys.exit("Error: In side mode, you must provide at least two directories")
+
+        tiles = tiles_from_slippy_map(args.dirs[0])
+
+        for tile, path in tqdm(list(tiles), desc="Compare", unit="image", ascii=True):
+
+            width, heigh = Image.open(path).size
+            side = Image.new(mode="RGB", size=(len(args.dirs) * width, height))
+
+            x, y, z = list(map(str, tile))
+
+            for i, dir in enumerate(args.dirs):
+                try:
+                    img = Image.open(os.path.join(dir, z, x, "{}.png".format(y)))
+                except:
+                    img = Image.open(os.path.join(dir, z, x, "{}.webp".format(y))).convert("RGB")
+
+                assert image.size == img.size
+                side.paste(img, box=(i * width, 0))
+
+            os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
+            path = os.path.join(args.out, z, x, "{}.webp".format(y))
+            side.save(path, optimize=True)
+
+    elif args.mode == "diff":
+        if not args.images or not args.labels or not args.masks or not args.dataset:
+            sys.exit("Error: in diff mode, you must provide images, labels and masks directories, and dataset path")
+
+        dataset = load_config(args.dataset)
+        classes = dataset["common"]["classes"]
+        colors = dataset["common"]["colors"]
+        assert len(classes) == len(colors), "classes and colors coincide"
+        assert len(colors) == 2, "only binary models supported right now"
+
+        palette_mask = make_palette(colors[0], colors[1])
+        palette_label = complementary_palette(palette_mask)
+
+        images = tiles_from_slippy_map(args.images)
+
+        for tile, path in tqdm(list(images), desc="Compare", unit="image", ascii=True):
+            x, y, z = list(map(str, tile))
+            os.makedirs(os.path.join(args.out, str(z), str(x)), exist_ok=True)
+
+            image = Image.open(path).convert("RGB")
+            label = Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))
+            mask = Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))
+
+            assert image.size == label.size == mask.size
+            assert label.getbands() == mask.getbands() == tuple("P")
 
-        image = Image.open(path).convert("RGB")
-        label = Image.open(os.path.join(args.labels, z, x, "{}.png".format(y))).convert("P")
-        assert image.size == label.size
+            dist, fg_ratio, qod = compare(np.array(mask), np.array(label), classes)
 
-        keep = False
-        masks = []
-        for path in args.masks:
-            mask = Image.open(os.path.join(path, z, x, "{}.png".format(y))).convert("P")
-            assert image.size == mask.size
-            masks.append(mask)
+            image = np.array(image) * 0.7
+            if args.minimum_fg < fg_ratio and qod < args.minimum_qod:
+                image += np.array(Image.new("RGB", label.size, (int(255 * 0.3), 0, 0)))
 
-            # TODO: The calculation below does not work for multi-class.
-            percentage = np.sum(np.array(mask) != 0) / np.prod(image.size)
+            mask.putpalette(palette_mask)
+            label.putpalette(palette_label)
+            mask = np.array(mask.convert("RGB"))
+            label = np.array(label.convert("RGB"))
 
-            # Keep this image when percentage is within required threshold.
-            if percentage >= args.minimum and percentage <= args.maximum:
-                keep = True
+            diff = Image.fromarray(np.uint8((image + mask + label) / 3.0))
+            diff.save(os.path.join(args.out, str(z), str(x), "{}.webp".format(y)), optimize=True)
 
-        if not keep:
-            continue
+        if args.leaflet:
+            tiles = [tile for tile, _ in tiles_from_slippy_map(args.images)]
+            leaflet(args.out, args.leaflet, tiles, "webp")
 
-        width, height = image.size
+    elif args.mode == "list":
+        if not args.labels or not args.masks or not args.dataset:
+            sys.exit("In list mode, you must provide labels and masks directories, and dataset path")
 
-        # Columns for image, label and all the masks.
-        columns = 2 + len(masks)
-        combined = Image.new(mode="RGB", size=(columns * width, height))
+        dataset = load_config(args.dataset)
+        masks = tiles_from_slippy_map(args.masks)
+        os.makedirs(os.path.basename(args.out), exist_ok=True)
+        log = Log(args.out, out=None, mode="w")
 
-        combined.paste(image, box=(0 * width, 0))
-        combined.paste(label, box=(1 * width, 0))
-        for i, mask in enumerate(masks):
-            combined.paste(mask, box=((2 + i) * width, 0))
+        for tile, path in tqdm(list(masks), desc="Compare", unit="image", ascii=True):
+            x, y, z = list(map(str, tile))
+            mask = Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))
+            label = Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))
 
-        os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
-        path = os.path.join(args.out, z, x, "{}.png".format(y))
-        combined.save(path, optimize=True)
+            assert label.size == mask.size
+            assert label.getbands() == mask.getbands() == tuple("P")
 
-    if args.leaflet:
-        leaflet(args.out, args.leaflet, images, ".png")
+            dist, fg_ratio, qod = compare(np.array(mask), np.array(label), dataset["common"]["classes"])
+            if args.minimum_fg < fg_ratio and qod < args.minimum_qod:
+                log.log("{},{},{}\t\t{:.3f}\t\t{:.3f}\t\t{:.3f}".format(x, y, z, dist, fg_ratio, qod))

From de654a43a8608d4ccdf1dded3b8094785c45f482 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 14:45:50 +0100
Subject: [PATCH 32/97] Remove rio-tiler overhead dependancy.

---
 deps/requirements-lock.txt |  1 -
 deps/requirements.txt      |  1 -
 robosat/tools/tile.py      | 23 +++++++++++++++++------
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 2a467791..3ee2687f 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -42,4 +42,3 @@ torchvision==0.2.1
 tqdm==4.23.4
 urllib3==1.22
 Werkzeug==0.14.1
-rio_tiler==1.0a7
diff --git a/deps/requirements.txt b/deps/requirements.txt
index 7a8b1bfb..4c5e6b72 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -17,4 +17,3 @@ rtree
 pyproj
 toml
 pytest
-rio_tiler
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 21e8fda1..dfa8babc 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -7,7 +7,10 @@
 from PIL import Image
 
 import mercantile
-from rio_tiler import main as tiler
+import rasterio as rio
+from rasterio.vrt import WarpedVRT
+from rasterio.warp import transform_bounds
+from rasterio.enums import Resampling
 from robosat.config import load_config
 from robosat.colors import make_palette
 from robosat.utils import leaflet
@@ -24,7 +27,7 @@ def add_parser(subparser):
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
-    parser.add_argument("--no_edges", action="store_true", help="skip to generate edges tiles")
+    parser.add_argument("--no_edges", action="store_true", help="don't generate edges tiles (to avoid black margins)")
     parser.add_argument("--label_threshold", type=int, default=1, help="label value threshold")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
@@ -44,8 +47,13 @@ def main(args):
         assert len(classes) == len(colors), "classes and colors coincide"
         assert len(colors) == 2, "only binary models supported right now"
 
-    bounds = tiler.bounds(args.raster)["bounds"]
-    tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(*bounds + [[args.zoom]])]
+    try:
+        raster = rio.open(args.raster)
+        warp_vrt = WarpedVRT(raster, dst_crs="EPSG:3857", resampling=Resampling.bilinear)
+        bounds = transform_bounds(*[raster.crs, "epsg:4326"] + list(raster.bounds))
+        tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(*bounds + (args.zoom,))]
+    except:
+        sys.exit("Error: Unable to load raster or deal with it's projection")
 
     if args.no_edges:
         edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
@@ -57,8 +65,11 @@ def main(args):
 
         os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
         path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
-        data = tiler.tile(args.raster, tile.x, tile.y, args.zoom, tilesize=args.size)[0]
-
+        data = warp_vrt.read(
+            out_shape=(len(raster.indexes), args.size, args.size),
+            window=warp_vrt.window(*mercantile.xy_bounds(tile)),
+            boundless=True,
+        )
         C, W, H = data.shape
 
         if args.type == "label":

From bf30a22604c524f89ebf3e518b01e1ea56c4997b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 30 Oct 2018 18:49:36 +0100
Subject: [PATCH 33/97] Use the latest rasterio version, and deal with
 boundless vrt issue, in tile tool.

---
 deps/requirements-lock.txt |  2 +-
 robosat/tools/tile.py      | 25 ++++++++++++++++++-------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 3ee2687f..4f4ce93e 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -28,7 +28,7 @@ pyproj==1.9.5.1
 pytest==3.6.1
 python-dateutil==2.7.3
 pytz==2018.4
-rasterio==1.0b1
+rasterio==1.0.9
 requests==2.18.4
 Rtree==0.8.3
 scipy==1.1.0
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index dfa8babc..7e0f186a 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import math
 import argparse
 from tqdm import tqdm
 
@@ -9,8 +10,8 @@
 import mercantile
 import rasterio as rio
 from rasterio.vrt import WarpedVRT
-from rasterio.warp import transform_bounds
 from rasterio.enums import Resampling
+from rasterio.warp import transform_bounds, calculate_default_transform
 from robosat.config import load_config
 from robosat.colors import make_palette
 from robosat.utils import leaflet
@@ -49,8 +50,7 @@ def main(args):
 
     try:
         raster = rio.open(args.raster)
-        warp_vrt = WarpedVRT(raster, dst_crs="EPSG:3857", resampling=Resampling.bilinear)
-        bounds = transform_bounds(*[raster.crs, "epsg:4326"] + list(raster.bounds))
+        bounds = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
         tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(*bounds + (args.zoom,))]
     except:
         sys.exit("Error: Unable to load raster or deal with it's projection")
@@ -65,11 +65,22 @@ def main(args):
 
         os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
         path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
-        data = warp_vrt.read(
-            out_shape=(len(raster.indexes), args.size, args.size),
-            window=warp_vrt.window(*mercantile.xy_bounds(tile)),
-            boundless=True,
+
+        # Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
+        transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, *bounds)
+        w, s, e, n = tile_bounds = mercantile.xy_bounds(tile)
+
+        warp_vrt = WarpedVRT(
+            raster,
+            crs="EPSG:3857",
+            resampling=Resampling.bilinear,
+            add_alpha=False,
+            transform=rio.transform.from_bounds(*tile_bounds, args.size, args.size),
+            width=math.ceil((e - w) / transform.a),
+            height=math.ceil((s - n) / transform.e),
         )
+
+        data = warp_vrt.read(out_shape=(len(raster.indexes), args.size, args.size), window=warp_vrt.window(w, s, e, n))
         C, W, H = data.shape
 
         if args.type == "label":

From 173a60dbe255016663d95d4e7f8fcdb54d294971 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 5 Nov 2018 10:24:02 +0100
Subject: [PATCH 34/97] add slippy map option in cover tool

---
 robosat/tools/cover.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/robosat/tools/cover.py b/robosat/tools/cover.py
index 213ae346..1a20534d 100644
--- a/robosat/tools/cover.py
+++ b/robosat/tools/cover.py
@@ -7,6 +7,8 @@
 from mercantile import tiles
 from tqdm import tqdm
 
+from robosat.datasets import tiles_from_slippy_map
+
 
 def add_parser(subparser):
     parser = subparser.add_parser(
@@ -15,9 +17,10 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
+    parser.add_argument("--zoom", type=int, help="zoom level of tiles")
     parser.add_argument("--features", type=str, help="path to GeoJSON features")
     parser.add_argument("--bbox", type=str, help="bbox expressed in lat/lon (i.e EPSG:4326)")
+    parser.add_argument("--path", type=str, help="Slippy Map directory input path")
     parser.add_argument("out", type=str, help="path to csv file to store tiles in")
 
     parser.set_defaults(func=main)
@@ -25,6 +28,9 @@ def add_parser(subparser):
 
 def main(args):
 
+    if not args.zoom and (args.features or args.bbox):
+        sys.exit("Zoom parameter is mandatory")
+
     cover = []
 
     if args.features:
@@ -41,8 +47,11 @@ def main(args):
         west, south, east, north = map(float, args.bbox.split(","))
         cover = tiles(west, south, east, north, args.zoom)
 
+    elif args.path:
+        cover = [tile for tile, _ in tiles_from_slippy_map(args.path)]
+
     else:
-        sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox")
+        sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox, or an input directory path")
 
     with open(args.out, "w") as fp:
         writer = csv.writer(fp)

From 98efc71e15677b2ac1d7b6573bb66de8d00eaf63 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 6 Nov 2018 10:38:39 +0100
Subject: [PATCH 35/97] rather than removing edges, deal with no_data values to
 keep only the meaningfull tiles. remove reclass threshold stuff. Improve
 performances

---
 robosat/tools/tile.py | 44 ++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 7e0f186a..beed6597 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -8,10 +8,13 @@
 from PIL import Image
 
 import mercantile
-import rasterio as rio
+
+from rasterio import open as rasterio_open
 from rasterio.vrt import WarpedVRT
 from rasterio.enums import Resampling
 from rasterio.warp import transform_bounds, calculate_default_transform
+from rasterio.transform import from_bounds
+
 from robosat.config import load_config
 from robosat.colors import make_palette
 from robosat.utils import leaflet
@@ -28,8 +31,7 @@ def add_parser(subparser):
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
-    parser.add_argument("--no_edges", action="store_true", help="don't generate edges tiles (to avoid black margins)")
-    parser.add_argument("--label_threshold", type=int, default=1, help="label value threshold")
+    parser.add_argument("--no_data", type=int, help="color considered as no data [0-255]. Skip related tile")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
@@ -49,46 +51,50 @@ def main(args):
         assert len(colors) == 2, "only binary models supported right now"
 
     try:
-        raster = rio.open(args.raster)
-        bounds = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
-        tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(*bounds + (args.zoom,))]
+        os.environ["GDAL_CACHEMAX"] = "50%"  # rasterio Env don't (yet) handle % settings
+        raster = rasterio_open(args.raster)
+        w, s, e, n = bounds = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
+        transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, *bounds)
     except:
         sys.exit("Error: Unable to load raster or deal with it's projection")
 
-    if args.no_edges:
-        edges_x = (min(tiles, key=lambda xy: xy[0])[0]), (max(tiles, key=lambda xy: xy[0])[0])
-        edges_y = (min(tiles, key=lambda xy: xy[1])[1]), (max(tiles, key=lambda xy: xy[1])[1])
-        tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in tiles if x not in edges_x and y not in edges_y]
-        assert len(tiles), "Error: Nothing left to tile, once the edges removed"
+    tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)]
+    tiles_nodata = []
 
     for tile in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
 
         os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
         path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
 
-        # Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
-        transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, *bounds)
         w, s, e, n = tile_bounds = mercantile.xy_bounds(tile)
 
+        # Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
         warp_vrt = WarpedVRT(
             raster,
             crs="EPSG:3857",
             resampling=Resampling.bilinear,
             add_alpha=False,
-            transform=rio.transform.from_bounds(*tile_bounds, args.size, args.size),
+            transform=from_bounds(*tile_bounds, args.size, args.size),
             width=math.ceil((e - w) / transform.a),
             height=math.ceil((s - n) / transform.e),
         )
-
         data = warp_vrt.read(out_shape=(len(raster.indexes), args.size, args.size), window=warp_vrt.window(w, s, e, n))
+
+        # If no_data is set, remove all tiles with at least one whole border filled only with no_data (on all bands)
+        if type(args.no_data) is not None and (
+            np.all(data[:, 0, :] == args.no_data)
+            or np.all(data[:, -1, :] == args.no_data)
+            or np.all(data[:, :, 0] == args.no_data)
+            or np.all(data[:, :, -1] == args.no_data)
+        ):
+            tiles_nodata.append(tile)
+            continue
+
         C, W, H = data.shape
 
         if args.type == "label":
             assert C == 1, "Error: Label raster input should be 1 band"
 
-            data[data < args.label_threshold] = 0
-            data[data >= args.label_threshold] = 1
-
             ext = "png"
             img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
             img.putpalette(make_palette(colors[0], colors[1]))
@@ -114,4 +120,4 @@ def main(args):
             sys.exit("Error: Unknown type, should be either 'image' or 'label'")
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles, ext)
+        leaflet(args.out, args.leaflet, [tile for tile in tiles if tile not in tiles_nodata], ext)

From 350d46f2e43de41be95eb23adfc3310621f6eeda Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 6 Nov 2018 10:58:33 +0100
Subject: [PATCH 36/97] Give a human a chance to read it back once black
 formatted ^^

---
 robosat/tiles.py | 51 +++++++++++++++++-------------------------------
 1 file changed, 18 insertions(+), 33 deletions(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index 93bad0ea..ce6c2e29 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -203,43 +203,28 @@ def buffer_tile_image(tile, tiles, overlap, tile_size, nodata=0):
     center = Image.open(path).convert("RGB")
     composite.paste(center, box=(overlap, overlap))
 
-    top_left = adjacent_tile(tile, -1, -1, tiles)
-    top_right = adjacent_tile(tile, +1, -1, tiles)
-    bottom_left = adjacent_tile(tile, -1, +1, tiles)
-    bottom_right = adjacent_tile(tile, +1, +1, tiles)
-
-    top = adjacent_tile(tile, 0, -1, tiles)
-    left = adjacent_tile(tile, -1, 0, tiles)
-    bottom = adjacent_tile(tile, 0, +1, tiles)
-    right = adjacent_tile(tile, +1, 0, tiles)
+    # 3x3 matrix (upper, center, bottom) x (left, center, right)
+    ul = adjacent_tile(tile, -1, -1, tiles)
+    ur = adjacent_tile(tile, +1, -1, tiles)
+    bl = adjacent_tile(tile, -1, +1, tiles)
+    br = adjacent_tile(tile, +1, +1, tiles)
+    uc = adjacent_tile(tile, +0, -1, tiles)
+    cl = adjacent_tile(tile, -1, +0, tiles)
+    bc = adjacent_tile(tile, +0, +1, tiles)
+    cr = adjacent_tile(tile, +1, +0, tiles)
 
     def maybe_stitch(maybe_tile, composite_box, tile_box):
         if maybe_tile:
             stitch_image(composite, composite_box, maybe_tile, tile_box)
 
-    maybe_stitch(top_left, (0, 0, overlap, overlap), (tile_size - overlap, tile_size - overlap, tile_size, tile_size))
-    maybe_stitch(
-        top_right, (tile_size + overlap, 0, composite_size, overlap), (0, tile_size - overlap, overlap, tile_size)
-    )
-    maybe_stitch(
-        bottom_left,
-        (0, composite_size - overlap, overlap, composite_size),
-        (tile_size - overlap, 0, tile_size, overlap),
-    )
-    maybe_stitch(
-        bottom_right,
-        (composite_size - overlap, composite_size - overlap, composite_size, composite_size),
-        (0, 0, overlap, overlap),
-    )
-    maybe_stitch(top, (overlap, 0, composite_size - overlap, overlap), (0, tile_size - overlap, tile_size, tile_size))
-    maybe_stitch(left, (0, overlap, overlap, composite_size - overlap), (tile_size - overlap, 0, tile_size, tile_size))
-    maybe_stitch(
-        bottom,
-        (overlap, composite_size - overlap, composite_size - overlap, composite_size),
-        (0, 0, tile_size, overlap),
-    )
-    maybe_stitch(
-        right, (composite_size - overlap, overlap, composite_size, composite_size - overlap), (0, 0, overlap, tile_size)
-    )
+    o = overlap
+    maybe_stitch(ul, (0, 0, o, o), (tile_size - o, tile_size - o, tile_size, tile_size))
+    maybe_stitch(ur, (tile_size + o, 0, composite_size, o), (0, tile_size - o, o, tile_size))
+    maybe_stitch(bl, (0, composite_size - o, o, composite_size), (tile_size - o, 0, tile_size, o))
+    maybe_stitch(br, (composite_size - o, composite_size - o, composite_size, composite_size), (0, 0, o, o))
+    maybe_stitch(uc, (o, 0, composite_size - o, o), (0, tile_size - o, tile_size, tile_size))
+    maybe_stitch(cl, (0, o, o, composite_size - o), (tile_size - o, 0, tile_size, tile_size))
+    maybe_stitch(bc, (o, composite_size - o, composite_size - o, composite_size), (0, 0, tile_size, o))
+    maybe_stitch(cr, (composite_size - o, o, composite_size, composite_size - o), (0, 0, o, tile_size))
 
     return composite

From 7a5eae450620412d57a32093a84c1f456495641c Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 09:14:11 +0000
Subject: [PATCH 37/97] revert perf setting, as they lead to issues on some
 platforms. Deployment in mind

---
 deps/requirements-lock.txt | 2 +-
 deps/requirements.txt      | 4 ++--
 robosat/tools/tile.py      | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 4f4ce93e..df17af10 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -20,7 +20,7 @@ more-itertools==4.2.0
 numpy==1.14.4
 opencv-contrib-python==3.4.1.15
 osmium==2.14.1
-Pillow-simd==5.3.0.post0
+Pillow==5.3.0
 pluggy==0.6.0
 py==1.5.3
 pyparsing==2.2.0
diff --git a/deps/requirements.txt b/deps/requirements.txt
index 4c5e6b72..1c403232 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -1,6 +1,6 @@
 torchvision
 numpy
-pillow-simd
+pillow
 scipy
 opencv-contrib-python
 tqdm
@@ -10,7 +10,7 @@ geojson
 mercantile
 osmium
 matplotlib
-rasterio>=1.0a12
+rasterio
 supermercado
 shapely
 rtree
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index beed6597..e563a47e 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -51,7 +51,6 @@ def main(args):
         assert len(colors) == 2, "only binary models supported right now"
 
     try:
-        os.environ["GDAL_CACHEMAX"] = "50%"  # rasterio Env don't (yet) handle % settings
         raster = rasterio_open(args.raster)
         w, s, e, n = bounds = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
         transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, *bounds)

From 275f017ae00b9b30bc70e4504780ba2ca0989b9f Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 09:44:09 +0000
Subject: [PATCH 38/97] Use cuda if available, and so no model config file
 needed anymore

---
 robosat/tools/predict.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 1c8bfc94..e0b7f825 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -36,7 +36,6 @@ def add_parser(subparser):
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
     parser.add_argument("tiles", type=str, help="directory to read slippy map image tiles from")
     parser.add_argument("probs", type=str, help="directory to save slippy map probability masks to")
-    parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
@@ -45,20 +44,17 @@ def add_parser(subparser):
 
 
 def main(args):
-    model = load_config(args.model)
     dataset = load_config(args.dataset)
+    num_classes = len(dataset["common"]["classes"])
 
-    cuda = model["common"]["cuda"]
-
-    device = torch.device("cuda" if cuda else "cpu")
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+        torch.backends.cudnn.benchmark = True
+    else:
+        device = torch.device("cpu")
 
     def map_location(storage, _):
-        return storage.cuda() if cuda else storage.cpu()
-
-    if cuda and not torch.cuda.is_available():
-        sys.exit("Error: CUDA requested but not available")
-
-    num_classes = len(dataset["common"]["classes"])
+        return storage.cuda() if torch.cuda.is_available() else storage.cpu()
 
     # https://github.com/pytorch/pytorch/issues/7178
     chkpt = torch.load(args.checkpoint, map_location=map_location)
@@ -66,9 +62,6 @@ def map_location(storage, _):
     net = UNet(num_classes).to(device)
     net = nn.DataParallel(net)
 
-    if cuda:
-        torch.backends.cudnn.benchmark = True
-
     net.load_state_dict(chkpt["state_dict"])
     net.eval()
 

From 47364ea69d92d5de1308956a12e9f1c2f5815e7f Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 10:20:51 +0000
Subject: [PATCH 39/97] Avoid to recompute palette on each tile generation

---
 robosat/tools/predict.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index e0b7f825..520a473b 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -72,6 +72,8 @@ def map_location(storage, _):
     directory = BufferedSlippyMapDirectory(args.tiles, transform=transform, size=args.tile_size, overlap=args.overlap)
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
 
+    palette = make_palette("denim", "orange") if args.masks_output else continuous_palette_for_color("pink", 256)
+
     # don't track tensors with autograd during prediction
     with torch.no_grad():
         for images, tiles in tqdm(loader, desc="Eval", unit="batch", ascii=True):
@@ -91,18 +93,9 @@ def map_location(storage, _):
                 assert np.allclose(np.sum(prob, axis=0), 1.), "single channel requires probabilities to sum up to one"
 
                 foreground = prob[1:, :, :]
+                image = np.around(foreground) if args.masks_output else np.digitize(foreground, np.linspace(0, 1, 256))
 
-                if args.masks_output:
-                    # Quantize the floating point prob in [0,1] to {0,1} with a fixed palette attached
-                    image = np.around(foreground)
-                    palette = make_palette("denim", "orange")
-
-                else:
-                    # Quantize the floating point prob in [0,1] to [0,255] with a continuous color palette attached
-                    image = np.digitize(foreground, np.linspace(0, 1, 256))
-                    palette = continuous_palette_for_color("pink", 256)
-
-                out = Image.fromarray(image.squeeze().astype(np.uint8), mode="P")
+                out = Image.fromarray(image.squeeze().astype("uint8"), mode="P")
                 out.putpalette(palette)
 
                 os.makedirs(os.path.join(args.probs, str(z), str(x)), exist_ok=True)

From 9b0588cb779803c449d2efd0b7c890059b1a92a0 Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 10:32:12 +0000
Subject: [PATCH 40/97] black and clean stuff

---
 robosat/tools/predict.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 520a473b..097f2e35 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -90,12 +90,12 @@ def map_location(storage, _):
                 prob = directory.unbuffer(prob)
 
                 assert prob.shape[0] == 2, "single channel requires binary model"
-                assert np.allclose(np.sum(prob, axis=0), 1.), "single channel requires probabilities to sum up to one"
+                assert np.allclose(np.sum(prob, axis=0), 1.0), "single channel requires probabilities to sum up to one"
 
                 foreground = prob[1:, :, :]
                 image = np.around(foreground) if args.masks_output else np.digitize(foreground, np.linspace(0, 1, 256))
 
-                out = Image.fromarray(image.squeeze().astype("uint8"), mode="P")
+                out = Image.fromarray(image.squeeze().astype(np.uint8), mode="P")
                 out.putpalette(palette)
 
                 os.makedirs(os.path.join(args.probs, str(z), str(x)), exist_ok=True)

From 94578807675f655b23be78e7c0334f166a57888e Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 20:35:21 +0000
Subject: [PATCH 41/97] switch from Pillow to OpenCV for predict. Refactor
 buffer tiling. Lead to perfs improvment

---
 robosat/colors.py        | 30 ++++++++++-----
 robosat/tiles.py         | 82 +++++++++++++++++-----------------------
 robosat/tools/predict.py | 18 +++++----
 3 files changed, 65 insertions(+), 65 deletions(-)

diff --git a/robosat/colors.py b/robosat/colors.py
index f0238fdb..f8c1abf5 100644
--- a/robosat/colors.py
+++ b/robosat/colors.py
@@ -2,6 +2,7 @@
 """
 
 import colorsys
+import numpy as np
 
 from enum import Enum, unique
 
@@ -42,16 +43,25 @@ class Mapbox(Enum):
     pink = _rgb("#ed6498")
 
 
-def make_palette(*colors):
+def make_palette(*colors, colormap=False):
     """Builds a PIL-compatible color palette from color names.
 
     Args:
       colors: variable number of color names.
     """
 
+    assert 0 < len(colors) <= 256
     rgbs = [Mapbox[color].value for color in colors]
-    flattened = sum(rgbs, ())
-    return list(flattened)
+
+    if colormap:
+        palette = np.zeros((256, 1, 3), np.uint8)
+        for i in range(len(colors)):
+            r, g, b = rgbs[i]
+            palette[i, 0, :] = b, g, r
+    else:
+        palette = list(sum(rgbs, ()))
+
+    return palette
 
 
 def color_string_to_rgb(color):
@@ -67,7 +77,7 @@ def color_string_to_rgb(color):
     return [*map(int, color.split(","))]
 
 
-def continuous_palette_for_color(color, bins=256):
+def continuous_palette_for_color(color, bins=256, colormap=False):
     """Creates a continuous color palette based on a single color.
 
     Args:
@@ -84,13 +94,15 @@ def continuous_palette_for_color(color, bins=256):
     r, g, b = [v / 255 for v in Mapbox[color].value]
     h, s, v = colorsys.rgb_to_hsv(r, g, b)
 
-    palette = []
+    assert 0 < bins <= 256
+    palette = np.zeros((256, 1, 3), np.uint8) if colormap else list()
 
     for i in range(bins):
-        ns = (1 / bins) * (i + 1)
-        palette.extend([int(v * 255) for v in colorsys.hsv_to_rgb(h, ns, v)])
-
-    assert len(palette) // 3 == bins
+        r, g, b = [int(v * 255) for v in colorsys.hsv_to_rgb(h, (1 / bins) * (i + 1), v)]
+        if colormap:
+            palette[i, 0, :] = b, g, r
+        else:
+            palette.extend(r, g, b)
 
     return palette
 
diff --git a/robosat/tiles.py b/robosat/tiles.py
index ce6c2e29..a80419de 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -12,7 +12,9 @@
 import io
 import os
 
-from PIL import Image
+import cv2
+import numpy as np
+
 from rasterio.warp import transform
 from rasterio.crs import CRS
 import mercantile
@@ -138,22 +140,6 @@ def tiles_from_csv(path):
             yield mercantile.Tile(*map(int, row))
 
 
-def stitch_image(into, into_box, image, image_box):
-    """Stitches two images together in-place.
-
-    Args:
-      into: the image to stitch into and modify in-place.
-      into_box: left, upper, right, lower image coordinates for where to place `image` in `into`.
-      image: the image to stitch into `into`.
-      image_box: left, upper, right, lower image coordinates for where to extract the sub-image from `image`.
-
-    Note:
-      Both boxes must be of same size.
-    """
-
-    into.paste(image.crop(box=image_box), box=into_box)
-
-
 def adjacent_tile(tile, dx, dy, tiles):
     """Retrieves an adjacent tile from a tile store.
 
@@ -168,16 +154,17 @@ def adjacent_tile(tile, dx, dy, tiles):
     """
 
     x, y, z = map(int, [tile.x, tile.y, tile.z])
-    other = mercantile.Tile(x=x + dx, y=y + dy, z=z)
+    adjacent = mercantile.Tile(x=x + dx, y=y + dy, z=z)
 
     try:
-        path = tiles[other]
-        return Image.open(path).convert("RGB")
+        path = tiles[adjacent]
     except KeyError:
         return None
 
+    return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
+
 
-def buffer_tile_image(tile, tiles, overlap, tile_size, nodata=0):
+def buffer_tile_image(tile, tiles, overlap, tile_size):
     """Buffers a tile image adding borders on all sides based on adjacent tiles.
 
     Args:
@@ -185,46 +172,45 @@ def buffer_tile_image(tile, tiles, overlap, tile_size, nodata=0):
       tiles: available tiles; must be a mapping of tiles to their filesystem paths.
       overlap: the tile border to add on every side; in pixel.
       tile_size: the tile size.
-      nodata: the color value to use when no adjacent tile is available.
 
     Returns:
       The composite image containing the original tile plus tile overlap on all sides.
       It's size is `tile_size` + 2 * `overlap` pixel for each side.
     """
 
+    assert 0 <= overlap <= tile_size, "Overlap value can't be either negative or bigger than tile_size"
+
     tiles = dict(tiles)
     x, y, z = map(int, [tile.x, tile.y, tile.z])
 
-    # Todo: instead of nodata we should probably mirror the center image
-    composite_size = tile_size + 2 * overlap
-    composite = Image.new(mode="RGB", size=(composite_size, composite_size), color=nodata)
-
-    path = tiles[tile]
-    center = Image.open(path).convert("RGB")
-    composite.paste(center, box=(overlap, overlap))
-
     # 3x3 matrix (upper, center, bottom) x (left, center, right)
     ul = adjacent_tile(tile, -1, -1, tiles)
-    ur = adjacent_tile(tile, +1, -1, tiles)
-    bl = adjacent_tile(tile, -1, +1, tiles)
-    br = adjacent_tile(tile, +1, +1, tiles)
     uc = adjacent_tile(tile, +0, -1, tiles)
+    ur = adjacent_tile(tile, +1, -1, tiles)
     cl = adjacent_tile(tile, -1, +0, tiles)
-    bc = adjacent_tile(tile, +0, +1, tiles)
+    cc = adjacent_tile(tile, +0, +0, tiles)
     cr = adjacent_tile(tile, +1, +0, tiles)
+    bl = adjacent_tile(tile, -1, +1, tiles)
+    bc = adjacent_tile(tile, +0, +1, tiles)
+    br = adjacent_tile(tile, +1, +1, tiles)
 
-    def maybe_stitch(maybe_tile, composite_box, tile_box):
-        if maybe_tile:
-            stitch_image(composite, composite_box, maybe_tile, tile_box)
-
+    ts = tile_size
     o = overlap
-    maybe_stitch(ul, (0, 0, o, o), (tile_size - o, tile_size - o, tile_size, tile_size))
-    maybe_stitch(ur, (tile_size + o, 0, composite_size, o), (0, tile_size - o, o, tile_size))
-    maybe_stitch(bl, (0, composite_size - o, o, composite_size), (tile_size - o, 0, tile_size, o))
-    maybe_stitch(br, (composite_size - o, composite_size - o, composite_size, composite_size), (0, 0, o, o))
-    maybe_stitch(uc, (o, 0, composite_size - o, o), (0, tile_size - o, tile_size, tile_size))
-    maybe_stitch(cl, (0, o, o, composite_size - o), (tile_size - o, 0, tile_size, tile_size))
-    maybe_stitch(bc, (o, composite_size - o, composite_size - o, composite_size), (0, 0, tile_size, o))
-    maybe_stitch(cr, (composite_size - o, o, composite_size, composite_size - o), (0, 0, o, tile_size))
-
-    return composite
+    oo = overlap * 2
+
+    # Todo: instead of nodata we should probably mirror the center image
+    img = np.zeros((ts + oo, ts + oo, 3)).astype(np.uint8)
+
+    # fmt:off
+    img[0:o,        0:o,        :] = ul[-o:ts, -o:ts, :] if ul is not None else np.zeros((o,   o, 3)).astype(np.uint8)
+    img[0:o,        o:ts+o,     :] = uc[-o:ts,  0:ts, :] if uc is not None else np.zeros((o,  ts, 3)).astype(np.uint8)
+    img[0:o,        ts+o:ts+oo, :] = ur[-o:ts,   0:o, :] if ur is not None else np.zeros((o,   o, 3)).astype(np.uint8)
+    img[o:ts+o,     0:o,        :] = cl[0:ts,  -o:ts, :] if cl is not None else np.zeros((ts,  o, 3)).astype(np.uint8)
+    img[o:ts+o,     o:ts+o,     :] = cc                  if cc is not None else np.zeros((ts, ts, 3)).astype(np.uint8)
+    img[o:ts+o,     ts+o:ts+oo, :] = cr[0:ts,    0:o, :] if cr is not None else np.zeros((ts,  o, 3)).astype(np.uint8)
+    img[ts+o:ts+oo, 0:o,        :] = bl[0:o,   -o:ts, :] if bl is not None else np.zeros((o,   o, 3)).astype(np.uint8)
+    img[ts+o:ts+oo, o:ts+o,     :] = bc[0:o,    0:ts, :] if bc is not None else np.zeros((o,  ts, 3)).astype(np.uint8)
+    img[ts+o:ts+oo, ts+o:ts+oo, :] = br[0:o,     0:o, :] if br is not None else np.zeros((o,   o, 3)).astype(np.uint8)
+    # fmt:on
+
+    return img
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 097f2e35..a8ab0713 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -11,7 +11,7 @@
 from torchvision.transforms import Compose, Normalize
 
 from tqdm import tqdm
-from PIL import Image
+from cv2 import imwrite, applyColorMap
 
 from robosat.datasets import BufferedSlippyMapDirectory
 from robosat.tiles import tiles_from_slippy_map
@@ -72,7 +72,10 @@ def map_location(storage, _):
     directory = BufferedSlippyMapDirectory(args.tiles, transform=transform, size=args.tile_size, overlap=args.overlap)
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
 
-    palette = make_palette("denim", "orange") if args.masks_output else continuous_palette_for_color("pink", 256)
+    if args.masks_output:
+        colormap = make_palette("white", "pink", colormap=True)
+    else:
+        colormap = continuous_palette_for_color("pink", 256, colormap=True)
 
     # don't track tensors with autograd during prediction
     with torch.no_grad():
@@ -92,16 +95,15 @@ def map_location(storage, _):
                 assert prob.shape[0] == 2, "single channel requires binary model"
                 assert np.allclose(np.sum(prob, axis=0), 1.0), "single channel requires probabilities to sum up to one"
 
-                foreground = prob[1:, :, :]
-                image = np.around(foreground) if args.masks_output else np.digitize(foreground, np.linspace(0, 1, 256))
-
-                out = Image.fromarray(image.squeeze().astype(np.uint8), mode="P")
-                out.putpalette(palette)
+                if args.masks_output:
+                    image = np.around(prob[1:, :, :]).astype(np.uint8).squeeze()
+                else:
+                    image = (prob[1:, :, :] * 255).astype(np.uint8).squeeze()
 
                 os.makedirs(os.path.join(args.probs, str(z), str(x)), exist_ok=True)
                 path = os.path.join(args.probs, str(z), str(x), str(y) + ".png")
 
-                out.save(path, optimize=True)
+                imwrite(path, applyColorMap(image, colormap))
 
     if args.leaflet:
         leaflet(args.probs, args.leaflet, [tile for tile, _ in tiles_from_slippy_map(args.tiles)], "png")

From f9e788b32ad661d9af973f583bbf42f3b4e5723d Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 9 Nov 2018 22:00:30 +0100
Subject: [PATCH 42/97] Avoid to create directory if not sure to write tile in
 it...

---
 robosat/tools/tile.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index beed6597..153924c4 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -63,9 +63,6 @@ def main(args):
 
     for tile in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
 
-        os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
-        path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
-
         w, s, e, n = tile_bounds = mercantile.xy_bounds(tile)
 
         # Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
@@ -92,6 +89,9 @@ def main(args):
 
         C, W, H = data.shape
 
+        os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True)
+        path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
+
         if args.type == "label":
             assert C == 1, "Error: Label raster input should be 1 band"
 

From 2270d49ace121a7a31b771eece11d9e51a35b3b7 Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 9 Nov 2018 23:17:42 +0000
Subject: [PATCH 43/97] remove cuda parameter in config model. Use all visible
 GPUs, and if none CPU. Slightly improve train log

---
 config/model-unet.toml |  3 ---
 robosat/tools/serve.py | 19 ++++---------------
 robosat/tools/train.py | 23 ++++++++++++-----------
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 9724c208..5c4f3c20 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -5,9 +5,6 @@
 # Model specific common attributes.
 [common]
 
-  # Use CUDA for GPU acceleration.
-  cuda       = true
-
   # Batch size for training.
   batch_size = 2
 
diff --git a/robosat/tools/serve.py b/robosat/tools/serve.py
index 28aaa651..ea23cab5 100644
--- a/robosat/tools/serve.py
+++ b/robosat/tools/serve.py
@@ -84,7 +84,6 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
 
     parser.add_argument("--url", type=str, help="endpoint with {z}/{x}/{y} variables to fetch image tiles from")
@@ -97,14 +96,8 @@ def add_parser(subparser):
 
 
 def main(args):
-    model = load_config(args.model)
     dataset = load_config(args.dataset)
 
-    cuda = model["common"]["cuda"]
-
-    if cuda and not torch.cuda.is_available():
-        sys.exit("Error: CUDA requested but not available")
-
     global size
     size = args.tile_size
 
@@ -121,7 +114,7 @@ def main(args):
     tiles = args.url
 
     global predictor
-    predictor = Predictor(args.checkpoint, model, dataset)
+    predictor = Predictor(args.checkpoint, dataset)
 
     app.run(host=args.host, port=args.port, threaded=False)
 
@@ -134,16 +127,12 @@ def send_png(image):
 
 
 class Predictor:
-    def __init__(self, checkpoint, model, dataset):
-        cuda = model["common"]["cuda"]
-
-        assert torch.cuda.is_available() or not cuda, "cuda is available when requested"
+    def __init__(self, checkpoint, dataset):
 
-        self.cuda = cuda
-        self.device = torch.device("cuda" if cuda else "cpu")
+        self.cuda = torch.cuda.is_available()
+        self.device = torch.device("cuda" if self.cuda else "cpu")
 
         self.checkpoint = checkpoint
-        self.model = model
         self.dataset = dataset
 
         self.net = self.net_from_chkpt_()
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 32c1bf0a..540d55d3 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -56,13 +56,15 @@ def main(args):
     model = load_config(args.model)
     dataset = load_config(args.dataset)
 
-    device = torch.device("cuda" if model["common"]["cuda"] else "cpu")
-
-    if model["common"]["cuda"] and not torch.cuda.is_available():
-        sys.exit("Error: CUDA requested but not available")
+    log = Log(os.path.join(model["common"]["checkpoint"], "log"))
 
-    # if args.batch_size < 2:
-    #     sys.exit('Error: PSPNet requires more than one image for BatchNorm in Pyramid Pooling')
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+        torch.backends.cudnn.benchmark = True
+        log.log("RoboSat - training on {} GPUs, with {} workers".format(torch.cuda.device_count(), args.workers))
+    else:
+        device = torch.device("cpu")
+        log.log("RoboSat - training on CPU, with {} workers", format(args.workers))
 
     os.makedirs(model["common"]["checkpoint"], exist_ok=True)
 
@@ -71,9 +73,6 @@ def main(args):
     net = DataParallel(net)
     net = net.to(device)
 
-    if model["common"]["cuda"]:
-        torch.backends.cudnn.benchmark = True
-
     if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
         try:
             weight = torch.Tensor(dataset["weights"]["values"])
@@ -86,11 +85,12 @@ def main(args):
     if args.checkpoint:
 
         def map_location(storage, _):
-            return storage.cuda() if model["common"]["cuda"] else storage.cpu()
+            return storage.cuda() if torch.cuda.is_available() else storage.cpu()
 
         # https://github.com/pytorch/pytorch/issues/7178
         chkpt = torch.load(args.checkpoint, map_location=map_location)
         net.load_state_dict(chkpt["state_dict"])
+        log.log("Using checkpoint: {}".format(args.checkpoint))
 
         if args.resume:
             optimizer.load_state_dict(chkpt["optimizer"])
@@ -114,8 +114,8 @@ def map_location(storage, _):
         sys.exit("Error: Epoch {} set in {} already reached by the checkpoint provided".format(num_epochs, args.model))
 
     history = collections.defaultdict(list)
-    log = Log(os.path.join(model["common"]["checkpoint"], "log"))
 
+    log.log("")
     log.log("--- Hyper Parameters on Dataset: {} ---".format(dataset["common"]["dataset"]))
     log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
     log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
@@ -126,6 +126,7 @@ def map_location(storage, _):
     if "weight" in locals():
         log.log("Weights :\t\t {}".format(dataset["weights"]["values"]))
     log.log("---")
+    log.log("")
 
     for epoch in range(resume, num_epochs):
         log.log("Epoch: {}/{}".format(epoch + 1, num_epochs))

From b75c3b11f8f85e28aa45e5765939e22e8c068386 Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Sat, 10 Nov 2018 19:03:01 +0000
Subject: [PATCH 44/97] Step backward on switching all PIL to OpenCV, as still
 not found a right way to handle PNG Pallete (with OpenCV). Use pillow-simd
 for performances.

---
 deps/requirements-lock.txt |  2 +-
 deps/requirements.txt      |  2 +-
 robosat/colors.py          | 21 +++++----------------
 robosat/tiles.py           | 25 +++++++++++++------------
 robosat/tools/predict.py   | 11 +++++++----
 robosat/tools/train.py     | 12 +++++++-----
 6 files changed, 34 insertions(+), 39 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index df17af10..a33b6a10 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -20,7 +20,7 @@ more-itertools==4.2.0
 numpy==1.14.4
 opencv-contrib-python==3.4.1.15
 osmium==2.14.1
-Pillow==5.3.0
+Pillow-simd==5.3.0
 pluggy==0.6.0
 py==1.5.3
 pyparsing==2.2.0
diff --git a/deps/requirements.txt b/deps/requirements.txt
index 1c403232..a30503f8 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -1,6 +1,6 @@
 torchvision
 numpy
-pillow
+pillow-simd
 scipy
 opencv-contrib-python
 tqdm
diff --git a/robosat/colors.py b/robosat/colors.py
index f8c1abf5..fafbaa53 100644
--- a/robosat/colors.py
+++ b/robosat/colors.py
@@ -43,7 +43,7 @@ class Mapbox(Enum):
     pink = _rgb("#ed6498")
 
 
-def make_palette(*colors, colormap=False):
+def make_palette(*colors):
     """Builds a PIL-compatible color palette from color names.
 
     Args:
@@ -53,15 +53,7 @@ def make_palette(*colors, colormap=False):
     assert 0 < len(colors) <= 256
     rgbs = [Mapbox[color].value for color in colors]
 
-    if colormap:
-        palette = np.zeros((256, 1, 3), np.uint8)
-        for i in range(len(colors)):
-            r, g, b = rgbs[i]
-            palette[i, 0, :] = b, g, r
-    else:
-        palette = list(sum(rgbs, ()))
-
-    return palette
+    return list(sum(rgbs, ()))
 
 
 def color_string_to_rgb(color):
@@ -77,7 +69,7 @@ def color_string_to_rgb(color):
     return [*map(int, color.split(","))]
 
 
-def continuous_palette_for_color(color, bins=256, colormap=False):
+def continuous_palette_for_color(color, bins=256):
     """Creates a continuous color palette based on a single color.
 
     Args:
@@ -95,14 +87,11 @@ def continuous_palette_for_color(color, bins=256, colormap=False):
     h, s, v = colorsys.rgb_to_hsv(r, g, b)
 
     assert 0 < bins <= 256
-    palette = np.zeros((256, 1, 3), np.uint8) if colormap else list()
 
+    palette = []
     for i in range(bins):
         r, g, b = [int(v * 255) for v in colorsys.hsv_to_rgb(h, (1 / bins) * (i + 1), v)]
-        if colormap:
-            palette[i, 0, :] = b, g, r
-        else:
-            palette.extend(r, g, b)
+        palette.extend(r, g, b)
 
     return palette
 
diff --git a/robosat/tiles.py b/robosat/tiles.py
index a80419de..dcd8c8fc 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -140,11 +140,11 @@ def tiles_from_csv(path):
             yield mercantile.Tile(*map(int, row))
 
 
-def adjacent_tile(tile, dx, dy, tiles):
-    """Retrieves an adjacent tile from a tile store.
+def adjacent_tile_image(tile, dx, dy, tiles):
+    """Retrieves an adjacent tile image from a tile store.
 
     Args:
-      tile: the original tile to get an adjacent tile for.
+      tile: the original tile to get an adjacent tile image for.
       dx: the offset in tile x direction.
       dy: the offset in tile y direction.
       tiles: the tile store to get tiles from; must support `__getitem__` with tiles.
@@ -161,6 +161,7 @@ def adjacent_tile(tile, dx, dy, tiles):
     except KeyError:
         return None
 
+    assert path[-5] == ".webp" # OpenCV AFAIK not handling PNG Palette
     return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
 
 
@@ -184,15 +185,15 @@ def buffer_tile_image(tile, tiles, overlap, tile_size):
     x, y, z = map(int, [tile.x, tile.y, tile.z])
 
     # 3x3 matrix (upper, center, bottom) x (left, center, right)
-    ul = adjacent_tile(tile, -1, -1, tiles)
-    uc = adjacent_tile(tile, +0, -1, tiles)
-    ur = adjacent_tile(tile, +1, -1, tiles)
-    cl = adjacent_tile(tile, -1, +0, tiles)
-    cc = adjacent_tile(tile, +0, +0, tiles)
-    cr = adjacent_tile(tile, +1, +0, tiles)
-    bl = adjacent_tile(tile, -1, +1, tiles)
-    bc = adjacent_tile(tile, +0, +1, tiles)
-    br = adjacent_tile(tile, +1, +1, tiles)
+    ul = adjacent_tile_image(tile, -1, -1, tiles)
+    uc = adjacent_tile_image(tile, +0, -1, tiles)
+    ur = adjacent_tile_image(tile, +1, -1, tiles)
+    cl = adjacent_tile_image(tile, -1, +0, tiles)
+    cc = adjacent_tile_image(tile, +0, +0, tiles)
+    cr = adjacent_tile_image(tile, +1, +0, tiles)
+    bl = adjacent_tile_image(tile, -1, +1, tiles)
+    bc = adjacent_tile_image(tile, +0, +1, tiles)
+    br = adjacent_tile_image(tile, +1, +1, tiles)
 
     ts = tile_size
     o = overlap
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index a8ab0713..446141f1 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -11,7 +11,7 @@
 from torchvision.transforms import Compose, Normalize
 
 from tqdm import tqdm
-from cv2 import imwrite, applyColorMap
+from PIL import Image
 
 from robosat.datasets import BufferedSlippyMapDirectory
 from robosat.tiles import tiles_from_slippy_map
@@ -73,9 +73,9 @@ def map_location(storage, _):
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
 
     if args.masks_output:
-        colormap = make_palette("white", "pink", colormap=True)
+        palette = make_palette("white", "pink")
     else:
-        colormap = continuous_palette_for_color("pink", 256, colormap=True)
+        palette = continuous_palette_for_color("pink", 256)
 
     # don't track tensors with autograd during prediction
     with torch.no_grad():
@@ -100,10 +100,13 @@ def map_location(storage, _):
                 else:
                     image = (prob[1:, :, :] * 255).astype(np.uint8).squeeze()
 
+                out = Image.fromarray(image, mode="P")
+                out.putpalette(palette)
+
                 os.makedirs(os.path.join(args.probs, str(z), str(x)), exist_ok=True)
                 path = os.path.join(args.probs, str(z), str(x), str(y) + ".png")
 
-                imwrite(path, applyColorMap(image, colormap))
+                out.save(path, optimize=True)
 
     if args.leaflet:
         leaflet(args.probs, args.leaflet, [tile for tile, _ in tiles_from_slippy_map(args.tiles)], "png")
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 540d55d3..86c36fdd 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -261,17 +261,19 @@ def get_dataset_loaders(model, dataset, workers):
         ]
     )
 
-    batch_size = model["common"]["batch_size"]
-    path = dataset["common"]["dataset"]
-
     train_dataset = SlippyMapTilesConcatenation(
-        os.path.join(path, "training", "images"), os.path.join(path, "training", "labels"), transform
+        os.path.join(dataset["common"]["dataset"], "training", "images"),
+        os.path.join(dataset["common"]["dataset"], "training", "labels"),
+        joint_transform=transform,
     )
 
     val_dataset = SlippyMapTilesConcatenation(
-        os.path.join(path, "validation", "images"), os.path.join(path, "validation", "labels"), transform
+        os.path.join(dataset["common"]["dataset"], "validation", "images"),
+        os.path.join(dataset["common"]["dataset"], "validation", "labels"),
+        joint_transform=transform,
     )
 
+    batch_size = model["common"]["batch_size"]
     train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=workers)
     val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, drop_last=True, num_workers=workers)
 

From 13d4f7cbaef5991d2305a1db5e3e81ef9cadca4a Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Sat, 10 Nov 2018 19:10:24 +0000
Subject: [PATCH 45/97] Better to write output dir before to try to log in...

---
 robosat/tools/train.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 86c36fdd..9c5826ad 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -56,6 +56,7 @@ def main(args):
     model = load_config(args.model)
     dataset = load_config(args.dataset)
 
+    os.makedirs(model["common"]["checkpoint"], exist_ok=True)
     log = Log(os.path.join(model["common"]["checkpoint"], "log"))
 
     if torch.cuda.is_available():
@@ -66,8 +67,6 @@ def main(args):
         device = torch.device("cpu")
         log.log("RoboSat - training on CPU, with {} workers", format(args.workers))
 
-    os.makedirs(model["common"]["checkpoint"], exist_ok=True)
-
     num_classes = len(dataset["common"]["classes"])
     net = UNet(num_classes)
     net = DataParallel(net)

From d260eef4e3a6158f2e622ab1f9d420ccac452580 Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Sat, 10 Nov 2018 19:51:15 +0000
Subject: [PATCH 46/97] typo

---
 robosat/tiles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index dcd8c8fc..c2391321 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -161,7 +161,7 @@ def adjacent_tile_image(tile, dx, dy, tiles):
     except KeyError:
         return None
 
-    assert path[-5] == ".webp" # OpenCV AFAIK not handling PNG Palette
+    assert path[-5:] == ".webp" # OpenCV AFAIK not handling PNG Palette
     return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
 
 

From 7c88599303f025fb0cf0c1743a2c12bfb1496ca3 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 10 Nov 2018 21:12:27 +0100
Subject: [PATCH 47/97] Fix issue if GPU available (with PyTorch 0.4.1)

---
 robosat/tools/export.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/robosat/tools/export.py b/robosat/tools/export.py
index 5a7ee5f3..d5311d30 100644
--- a/robosat/tools/export.py
+++ b/robosat/tools/export.py
@@ -1,5 +1,6 @@
 import argparse
 
+import os
 import torch
 import torch.onnx
 import torch.autograd
@@ -27,6 +28,7 @@ def main(args):
     num_classes = len(dataset["common"]["classes"])
     net = UNet(num_classes)
 
+    os.environ["CUDA_VISIBLE_DEVICES"] = "" # to be sure GPUs if any won't be used
     def map_location(storage, _):
         return storage.cpu()
 

From 7083cb947a8d6cc39612e42211965cc4174de5a2 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 11 Nov 2018 00:04:32 +0100
Subject: [PATCH 48/97] Upgrade to Torch 0.4.1 and switch to deprecated
 upsample to interpolate. Use deepcopy to avoid ONNX export issue. Unet num
 input channel not hard coded

---
 deps/requirements-lock.txt |  2 +-
 robosat/unet.py            | 15 +++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index a33b6a10..4d1b06d3 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -37,7 +37,7 @@ six==1.11.0
 snuggs==1.4.1
 supermercado==0.0.5
 toml==0.9.4
-torch==0.4.0
+torch==0.4.1
 torchvision==0.2.1
 tqdm==4.23.4
 urllib3==1.22
diff --git a/robosat/unet.py b/robosat/unet.py
index bedabea1..59f62f79 100644
--- a/robosat/unet.py
+++ b/robosat/unet.py
@@ -12,6 +12,7 @@
 import torch
 import torch.nn as nn
 
+from copy import deepcopy
 from torchvision.models import resnet50
 
 
@@ -70,7 +71,7 @@ def forward(self, x):
           The networks output tensor.
         """
 
-        return self.block(nn.functional.upsample(x, scale_factor=2, mode="nearest"))
+        return self.block(nn.functional.interpolate(x, scale_factor=2, mode="nearest"))
 
 
 class UNet(nn.Module):
@@ -79,7 +80,7 @@ class UNet(nn.Module):
        Also known as AlbuNet due to its inventor Alexander Buslaev.
     """
 
-    def __init__(self, num_classes, num_filters=32, pretrained=True):
+    def __init__(self, num_classes, num_filters=32, num_channels=3, pretrained=True):
         """Creates an `UNet` instance for semantic segmentation.
 
         Args:
@@ -89,12 +90,14 @@ def __init__(self, num_classes, num_filters=32, pretrained=True):
 
         super().__init__()
 
-        # Todo: make input channels configurable, not hard-coded to three channels for RGB
-
         self.resnet = resnet50(pretrained=pretrained)
 
-        # Access resnet directly in forward pass; do not store refs here due to
-        # https://github.com/pytorch/pytorch/issues/8392
+        # Give a look at https://github.com/pytorch/pytorch/issues/8392 for deepcopy stuff
+        self.enc0 = nn.Sequential(deepcopy(self.resnet.conv1), nn.Conv2d(num_channels, 64, 3, padding=1, bias=False))
+        self.enc1 = deepcopy(self.resnet.layer1)
+        self.enc2 = deepcopy(self.resnet.layer2)
+        self.enc3 = deepcopy(self.resnet.layer3)
+        self.enc4 = deepcopy(self.resnet.layer4)
 
         self.center = DecoderBlock(2048, num_filters * 8)
 

From aad94fc70b30520a2e65bb1d39e060ae08bde8c1 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 12 Nov 2018 15:13:43 +0100
Subject: [PATCH 49/97] Multibands handling (at last). First prune export
 feature. Improve train log (again). pretrained resnet as an userland
 hyperparameter

---
 config/dataset-parking.toml |  5 ++++
 robosat/datasets.py         | 47 +++++++++++++++++++++++++++++--------
 robosat/tools/export.py     | 32 +++++++++++++++++++------
 robosat/tools/train.py      | 21 ++++++++++-------
 robosat/transforms.py       |  4 ++--
 robosat/unet.py             | 22 ++++++++++-------
 6 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/config/dataset-parking.toml b/config/dataset-parking.toml
index d45d545a..3a0f564c 100644
--- a/config/dataset-parking.toml
+++ b/config/dataset-parking.toml
@@ -15,6 +15,11 @@
   # Note: available colors can be found in `robosat/colors.py`
   colors  = ['denim', 'orange']
 
+  # Input Tensor is composed by several images bands
+  # Channels configuration let your indicate wich dataset sub-directory and band are concerned
+  # Order is meaningful, syntax is: directory_name:band_number (1-n notation)
+  channels = ['images:1', 'images:2', 'images:3']
+
 
 # Dataset specific class weights computes on the training data.
 # Needed by 'mIoU' and 'CrossEntropy' losses to deal with unbalanced classes.
diff --git a/robosat/datasets.py b/robosat/datasets.py
index 2b1e4f2d..33399ad3 100644
--- a/robosat/datasets.py
+++ b/robosat/datasets.py
@@ -5,6 +5,8 @@
 See: http://pytorch.org/docs/0.3.1/data.html
 """
 
+import os
+import sys
 import torch
 from PIL import Image
 import torch.utils.data
@@ -38,6 +40,16 @@ def __getitem__(self, i):
         if self.mode == "image":
             image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
 
+        elif self.mode == "multibands":
+            image = cv2.imread(path, cv2.IMREAD_ANYCOLOR)
+            if len(image.shape) == 3 and image.shape[2] >= 3:
+                # FIXME Look twice to find an in-place way to perform a multiband BGR2RGB
+                g = image[:, :, 0]
+                image[:, :, 0] = image[:, :, 2]
+                image[:, :, 2] = g
+            else:
+                image = image.reshape(image.shape[0], image.shape[1], 1)
+
         elif self.mode == "mask":
             image = np.array(Image.open(path).convert("P"))
 
@@ -52,31 +64,46 @@ class SlippyMapTilesConcatenation(torch.utils.data.Dataset):
     """Dataset to concate multiple input images stored in slippy map format.
     """
 
-    def __init__(self, input, target, joint_transform=None):
+    def __init__(self, input_path, channels, target, joint_transform=None):
         super().__init__()
 
-        # No transformations in the `SlippyMapTiles` instead joint transformations in getitem
-        self.joint_transform = joint_transform
+        assert len(channels), "Channels configuration empty"
+
+        self.inputs = dict()
+        for channel in channels:
+            try:
+                name, band = channel.split(":")
+                self.inputs[name] = SlippyMapTiles(os.path.join(input_path, name), mode="multibands")
+            except:
+                sys.exit("Channels configuration issue")
 
         self.target = SlippyMapTiles(target, mode="mask")
-        self.input = SlippyMapTiles(input, mode="image")
+        self.channels = channels
 
-        assert len(self.input) == len(self.target), "same number of tiles in images and label"
+        # No transformations in the `SlippyMapTiles` instead joint transformations in getitem
+        self.joint_transform = joint_transform
 
     def __len__(self):
         return len(self.target)
 
     def __getitem__(self, i):
 
-        image, image_tile = self.input[i]
-        mask, mask_tile = self.target[i]
+        mask, tile = self.target[i]
 
-        assert image_tile == mask_tile, "image tile is the same as label tile"
+        for channel in self.channels:
+            try:
+                name, band = channel.split(":")
+                data, input_tile = self.inputs[name][i]
+                assert input_tile == tile
+                data = data[:, :, int(band) - 1].reshape(mask.shape[0], mask.shape[1], 1)
+                tensor = np.concatenate((tensor, data), axis=2) if "tensor" in locals() else data
+            except:
+                sys.exit("Unable to concatenate input Tensor")
 
         if self.joint_transform is not None:
-            image, mask = self.joint_transform(image, mask)
+            tensor, mask = self.joint_transform(tensor, mask)
 
-        return image, mask, image_tile[0]
+        return tensor, mask, tile
 
 
 # Todo: once we have the SlippyMapDataset this dataset should wrap
diff --git a/robosat/tools/export.py b/robosat/tools/export.py
index d5311d30..8761c17e 100644
--- a/robosat/tools/export.py
+++ b/robosat/tools/export.py
@@ -4,6 +4,7 @@
 import torch
 import torch.onnx
 import torch.autograd
+import torch.nn as nn
 
 from robosat.config import load_config
 from robosat.unet import UNet
@@ -11,13 +12,15 @@
 
 def add_parser(subparser):
     parser = subparser.add_parser(
-        "export", help="exports model in ONNX format", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        "export", help="exports or prunes model", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--export_channels", type=int, help="export channels to use (keep the first ones)")
+    parser.add_argument("--type", type=str, default="onnx", help="output type, i.e onnx or pth")
     parser.add_argument("--image_size", type=int, default=512, help="image size to use for model")
     parser.add_argument("--checkpoint", type=str, required=True, help="model checkpoint to load")
-    parser.add_argument("model", type=str, help="path to save ONNX GraphProto .pb model to")
+    parser.add_argument("out", type=str, help="path to save export model to")
 
     parser.set_defaults(func=main)
 
@@ -25,18 +28,33 @@ def add_parser(subparser):
 def main(args):
     dataset = load_config(args.dataset)
 
+    if args.type == "onnx":
+        os.environ["CUDA_VISIBLE_DEVICES"] = ""
+        # Workaround: PyTorch ONNX, DataParallel with GPU issue, cf https://github.com/pytorch/pytorch/issues/5315
+
     num_classes = len(dataset["common"]["classes"])
-    net = UNet(num_classes)
+    num_channels = len(dataset["common"]["channels"])
+    export_channels = num_channels if not args.export_channels else args.export_channels
+    assert num_channels >= export_channels, "Will be hard indeed, to export more channels than thoses dataset provide"
 
-    os.environ["CUDA_VISIBLE_DEVICES"] = "" # to be sure GPUs if any won't be used
     def map_location(storage, _):
         return storage.cpu()
 
+    net = UNet(num_classes, num_channels=num_channels).to("cpu")
     chkpt = torch.load(args.checkpoint, map_location=map_location)
     net = torch.nn.DataParallel(net)
     net.load_state_dict(chkpt["state_dict"])
 
-    # Todo: make input channels configurable, not hard-coded to three channels for RGB
-    batch = torch.autograd.Variable(torch.randn(1, 3, args.image_size, args.image_size))
+    if export_channels < num_channels:
+        weights = torch.zeros((64, export_channels, 7, 7))
+        weights.data = net.module.resnet.conv1.weight.data[:, :export_channels, :, :]
+        net.module.resnet.conv1 = nn.Conv2d(num_channels, 64, kernel_size=7, stride=2, padding=3, bias=False)
+        net.module.resnet.conv1.weight = nn.Parameter(weights)
+
+    if args.type == "onnx":
+        batch = torch.autograd.Variable(torch.randn(1, export_channels, args.image_size, args.image_size))
+        torch.onnx.export(net, batch, args.out)
 
-    torch.onnx.export(net, batch, args.model)
+    elif args.type == "pth":
+        states = {"epoch": chkpt["epoch"], "state_dict": net.state_dict(), "optimizer": chkpt["optimizer"]}
+        torch.save(states, args.out)
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 9c5826ad..96ef0faa 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -68,9 +68,9 @@ def main(args):
         log.log("RoboSat - training on CPU, with {} workers", format(args.workers))
 
     num_classes = len(dataset["common"]["classes"])
-    net = UNet(num_classes)
-    net = DataParallel(net)
-    net = net.to(device)
+    num_channels = len(dataset["common"]["channels"])
+    pretrained = model["opt"]["pretrained"]
+    net = DataParallel(UNet(num_classes, num_channels=num_channels, pretrained=pretrained)).to(device)
 
     if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
         try:
@@ -115,13 +115,18 @@ def map_location(storage, _):
     history = collections.defaultdict(list)
 
     log.log("")
-    log.log("--- Hyper Parameters on Dataset: {} ---".format(dataset["common"]["dataset"]))
+    log.log("--- Input tensor from Dataset: {} ---".format(dataset["common"]["dataset"]))
+    for i in range(len(dataset["common"]["channels"])):
+        log.log("Channel {}:\t\t {}".format(i + 1, dataset["common"]["channels"][i]))
+    log.log("")
+    log.log("--- Hyper Parameters ---")
     log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
     log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
     log.log("Data Augmentation:\t {}".format(model["opt"]["data_augmentation"]))
     log.log("Learning Rate:\t\t {}".format(model["opt"]["lr"]))
     log.log("Weight Decay:\t\t {}".format(model["opt"]["decay"]))
     log.log("Loss function:\t\t {}".format(model["opt"]["loss"]))
+    log.log("ResNet pre-trained:\t {}".format(model["opt"]["pretrained"]))
     if "weight" in locals():
         log.log("Weights :\t\t {}".format(dataset["weights"]["values"]))
     log.log("---")
@@ -158,9 +163,7 @@ def map_location(storage, _):
         plot(os.path.join(model["common"]["checkpoint"], visual), history)
 
         checkpoint = "checkpoint-{:05d}-of-{:05d}.pth".format(epoch + 1, num_epochs)
-
         states = {"epoch": epoch + 1, "state_dict": net.state_dict(), "optimizer": optimizer.state_dict()}
-
         torch.save(states, os.path.join(model["common"]["checkpoint"], checkpoint))
 
 
@@ -261,13 +264,15 @@ def get_dataset_loaders(model, dataset, workers):
     )
 
     train_dataset = SlippyMapTilesConcatenation(
-        os.path.join(dataset["common"]["dataset"], "training", "images"),
+        os.path.join(dataset["common"]["dataset"], "training"),
+        dataset["common"]["channels"],
         os.path.join(dataset["common"]["dataset"], "training", "labels"),
         joint_transform=transform,
     )
 
     val_dataset = SlippyMapTilesConcatenation(
-        os.path.join(dataset["common"]["dataset"], "validation", "images"),
+        os.path.join(dataset["common"]["dataset"], "validation"),
+        dataset["common"]["channels"],
         os.path.join(dataset["common"]["dataset"], "validation", "labels"),
         joint_transform=transform,
     )
diff --git a/robosat/transforms.py b/robosat/transforms.py
index cbb0d0f3..96fa27b2 100644
--- a/robosat/transforms.py
+++ b/robosat/transforms.py
@@ -8,7 +8,7 @@
 
 
 class ImageToTensor:
-    """Callable to convert a OpenCV H,W,C image into a PyTorch tensor.
+    """Callable to convert a NumPy H,W,C image into a PyTorch C,W,H tensor.
     """
 
     def __call__(self, image):
@@ -25,7 +25,7 @@ def __call__(self, image):
 
 
 class MaskToTensor:
-    """Callable to convert an OpenCV H,W image into a PyTorch tensor.
+    """Callable to convert a NumPy H,W image into a PyTorch tensor.
     """
 
     def __call__(self, mask):
diff --git a/robosat/unet.py b/robosat/unet.py
index 59f62f79..ba50aadf 100644
--- a/robosat/unet.py
+++ b/robosat/unet.py
@@ -80,24 +80,30 @@ class UNet(nn.Module):
        Also known as AlbuNet due to its inventor Alexander Buslaev.
     """
 
-    def __init__(self, num_classes, num_filters=32, num_channels=3, pretrained=True):
+    def __init__(self, num_classes, num_channels=3, num_filters=32, pretrained=True):
         """Creates an `UNet` instance for semantic segmentation.
 
         Args:
           num_classes: number of classes to predict.
-          pretrained: use ImageNet pre-trained backbone feature extractor
+          num_channels: number of inputs channels (e.g bands)
+          pretrained: use ImageNet pre-trained ResNet Encoder weights
         """
 
         super().__init__()
 
         self.resnet = resnet50(pretrained=pretrained)
 
-        # Give a look at https://github.com/pytorch/pytorch/issues/8392 for deepcopy stuff
-        self.enc0 = nn.Sequential(deepcopy(self.resnet.conv1), nn.Conv2d(num_channels, 64, 3, padding=1, bias=False))
-        self.enc1 = deepcopy(self.resnet.layer1)
-        self.enc2 = deepcopy(self.resnet.layer2)
-        self.enc3 = deepcopy(self.resnet.layer3)
-        self.enc4 = deepcopy(self.resnet.layer4)
+        assert num_channels
+
+        if num_channels != 3:
+            weights = nn.init.xavier_uniform_(torch.zeros((64, num_channels, 7, 7)))
+            if pretrained:
+                for c in range(min(num_channels, 3)):
+                    weights.data[:, c, :, :] = self.resnet.conv1.weight.data[:, c, :, :]
+            self.resnet.conv1 = nn.Conv2d(num_channels, 64, kernel_size=7, stride=2, padding=3, bias=False)
+            self.resnet.conv1.weight = nn.Parameter(weights)
+
+        # No encoder reference, give a look at https://github.com/pytorch/pytorch/issues/8392
 
         self.center = DecoderBlock(2048, num_filters * 8)
 

From 8f1faca4f1dd9dee3c8c894f0fb474651c4fd63f Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 12 Nov 2018 22:48:57 +0100
Subject: [PATCH 50/97] Improve multidataset configuration

---
 config/dataset-parking.toml | 13 ++++++++-----
 robosat/datasets.py         | 27 ++++++++++++---------------
 robosat/tools/train.py      | 17 +++++++++++------
 3 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/config/dataset-parking.toml b/config/dataset-parking.toml
index 3a0f564c..1e5fd081 100644
--- a/config/dataset-parking.toml
+++ b/config/dataset-parking.toml
@@ -15,14 +15,17 @@
   # Note: available colors can be found in `robosat/colors.py`
   colors  = ['denim', 'orange']
 
-  # Input Tensor is composed by several images bands
-  # Channels configuration let your indicate wich dataset sub-directory and band are concerned
-  # Order is meaningful, syntax is: directory_name:band_number (1-n notation)
-  channels = ['images:1', 'images:2', 'images:3']
-
 
 # Dataset specific class weights computes on the training data.
 # Needed by 'mIoU' and 'CrossEntropy' losses to deal with unbalanced classes.
 # Note: use `./rs weights -h` to compute these for new datasets.
 [weights]
   values = [1.6248, 5.762827]
+
+
+# Channels configuration let your indicate wich dataset sub-directory and bands to take as input
+# You could so, add several channels blocks to compose your input Tensor. Orders are meaningful.
+[[channels]]
+type = "file"
+sub = "images"
+bands = [1,2,3]
diff --git a/robosat/datasets.py b/robosat/datasets.py
index 33399ad3..1c7d22a2 100644
--- a/robosat/datasets.py
+++ b/robosat/datasets.py
@@ -47,8 +47,6 @@ def __getitem__(self, i):
                 g = image[:, :, 0]
                 image[:, :, 0] = image[:, :, 2]
                 image[:, :, 2] = g
-            else:
-                image = image.reshape(image.shape[0], image.shape[1], 1)
 
         elif self.mode == "mask":
             image = np.array(Image.open(path).convert("P"))
@@ -64,21 +62,18 @@ class SlippyMapTilesConcatenation(torch.utils.data.Dataset):
     """Dataset to concate multiple input images stored in slippy map format.
     """
 
-    def __init__(self, input_path, channels, target, joint_transform=None):
+    def __init__(self, path, channels, target, joint_transform=None):
         super().__init__()
 
         assert len(channels), "Channels configuration empty"
-
+        self.channels = channels
         self.inputs = dict()
+
         for channel in channels:
-            try:
-                name, band = channel.split(":")
-                self.inputs[name] = SlippyMapTiles(os.path.join(input_path, name), mode="multibands")
-            except:
-                sys.exit("Channels configuration issue")
+            for band in channel["bands"]:
+                self.inputs[channel["sub"]] = SlippyMapTiles(os.path.join(path, channel["sub"]), mode="multibands")
 
         self.target = SlippyMapTiles(target, mode="mask")
-        self.channels = channels
 
         # No transformations in the `SlippyMapTiles` instead joint transformations in getitem
         self.joint_transform = joint_transform
@@ -92,11 +87,13 @@ def __getitem__(self, i):
 
         for channel in self.channels:
             try:
-                name, band = channel.split(":")
-                data, input_tile = self.inputs[name][i]
-                assert input_tile == tile
-                data = data[:, :, int(band) - 1].reshape(mask.shape[0], mask.shape[1], 1)
-                tensor = np.concatenate((tensor, data), axis=2) if "tensor" in locals() else data
+                data, band_tile = self.inputs[channel["sub"]][i]
+                assert band_tile == tile
+
+                for band in channel["bands"]:
+                    data_band = data[:, :, int(band) - 1] if len(data.shape) == 3 else data_band
+                    data_band = data_band.reshape(mask.shape[0], mask.shape[1], 1)
+                    tensor = np.concatenate((tensor, data_band), axis=2) if "tensor" in locals() else data_band
             except:
                 sys.exit("Unable to concatenate input Tensor")
 
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 96ef0faa..9e3bec79 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -68,7 +68,9 @@ def main(args):
         log.log("RoboSat - training on CPU, with {} workers", format(args.workers))
 
     num_classes = len(dataset["common"]["classes"])
-    num_channels = len(dataset["common"]["channels"])
+    num_channels = 0
+    for channel in dataset["channels"]:
+        num_channels += len(channel["bands"])
     pretrained = model["opt"]["pretrained"]
     net = DataParallel(UNet(num_classes, num_channels=num_channels, pretrained=pretrained)).to(device)
 
@@ -116,8 +118,11 @@ def map_location(storage, _):
 
     log.log("")
     log.log("--- Input tensor from Dataset: {} ---".format(dataset["common"]["dataset"]))
-    for i in range(len(dataset["common"]["channels"])):
-        log.log("Channel {}:\t\t {}".format(i + 1, dataset["common"]["channels"][i]))
+    num_channel = 1
+    for channel in dataset["channels"]:
+        for band in channel["bands"]:
+            log.log("Channel {}:\t\t {}[band: {}]".format(num_channel, channel["sub"], band))
+            num_channel += 1
     log.log("")
     log.log("--- Hyper Parameters ---")
     log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
@@ -129,10 +134,10 @@ def map_location(storage, _):
     log.log("ResNet pre-trained:\t {}".format(model["opt"]["pretrained"]))
     if "weight" in locals():
         log.log("Weights :\t\t {}".format(dataset["weights"]["values"]))
-    log.log("---")
     log.log("")
 
     for epoch in range(resume, num_epochs):
+        log.log("---")
         log.log("Epoch: {}/{}".format(epoch + 1, num_epochs))
 
         train_hist = train(train_loader, num_classes, device, net, optimizer, criterion)
@@ -265,14 +270,14 @@ def get_dataset_loaders(model, dataset, workers):
 
     train_dataset = SlippyMapTilesConcatenation(
         os.path.join(dataset["common"]["dataset"], "training"),
-        dataset["common"]["channels"],
+        dataset["channels"],
         os.path.join(dataset["common"]["dataset"], "training", "labels"),
         joint_transform=transform,
     )
 
     val_dataset = SlippyMapTilesConcatenation(
         os.path.join(dataset["common"]["dataset"], "validation"),
-        dataset["common"]["channels"],
+        dataset["channels"],
         os.path.join(dataset["common"]["dataset"], "validation", "labels"),
         joint_transform=transform,
     )

From 5ade9f2734c765dd535d3d02f6d7039c6ab9ef53 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 13 Nov 2018 09:15:35 +0100
Subject: [PATCH 51/97] allow to unzoom, by providing a vector grid

---
 robosat/tools/templates/leaflet.html | 3 ++-
 robosat/utils.py                     | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/templates/leaflet.html b/robosat/tools/templates/leaflet.html
index 6f22ab2e..47121176 100644
--- a/robosat/tools/templates/leaflet.html
+++ b/robosat/tools/templates/leaflet.html
@@ -14,8 +14,9 @@
 <body>
   <div id="mapid" style="width:100%; height:100vh;"></div>
   <script>
-    var m = L.map("mapid", {zoomControl: false, boxZoom: false, scrollWheelZoom: false}).setView({{center}}, {{zoom}});
+    var m = L.map("mapid").setView({{center}}, {{zoom}});
     L.tileLayer("{{base_url}}/{z}/{x}/{y}.{{ext}}").addTo(m);
+    L.geoJSON({{grid}}, {style: {"color": "#cc0099", "opacity": 0.1, "fillOpacity": 0}}).addTo(m);
   </script>
 </body>
 </html>
diff --git a/robosat/utils.py b/robosat/utils.py
index 28400169..78c54a15 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -1,6 +1,8 @@
 import re
 import os 
 from robosat.tiles import pixel_to_location
+import mercantile
+import json
 import matplotlib
 
 matplotlib.use("Agg")
@@ -27,9 +29,12 @@ def plot(out, history):
 
 def leaflet(out, base_url, tiles, ext):
 
+    grid = json.dumps([mercantile.feature(tile) for tile in tiles])
+
     leaflet = open("./robosat/tools/templates/leaflet.html", "r").read()
     leaflet = re.sub("{{base_url}}", base_url, leaflet)
     leaflet = re.sub("{{ext}}", ext, leaflet)
+    leaflet = re.sub("{{grid}}", grid, leaflet)
 
     # Could surely be improve, but for now, took the first tile to center on
     tile = (list(tiles)[0])

From e6a5f876d2ce924042a2edb682f47b8c77dc0b41 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 13 Nov 2018 11:26:01 +0100
Subject: [PATCH 52/97] improve userland type value resilience

---
 robosat/tools/download.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index ad3b068d..53995e04 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -68,9 +68,10 @@ def worker(tile):
                 elif args.type == "WMS":
                     xmin, ymin, xmax, ymax = tile_to_bbox(tile)
                     url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
+                else:
+                    return tile, None, False
 
                 res = fetch_image(session, url, args.timeout)
-
                 if not res:
                     return tile, url, False
 

From 1bf590ac10c47fed22633bc2afa3517f87899882 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 13 Nov 2018 21:24:22 +0100
Subject: [PATCH 53/97] refactor GeoJson parsing. Handle cleanly Polygon,
 MultiPolygon, GeometryCollection and N-Dimensionnal coordinates

---
 robosat/tools/rasterize.py | 49 ++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 7f2482f2..32dbc8b8 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -14,12 +14,13 @@
 from rasterio.features import rasterize
 from rasterio.warp import transform
 from supermercado import burntiles
-from shapely.geometry import shape, mapping
+from shapely.geometry import mapping
 
 from robosat.config import load_config
 from robosat.colors import make_palette
 from robosat.tiles import tiles_from_csv
 from robosat.utils import leaflet
+from robosat.log import Log
 
 
 def add_parser(subparser):
@@ -96,20 +97,42 @@ def main(args):
 
     # Find all tiles the features cover and make a map object for quick lookup.
     feature_map = collections.defaultdict(list)
+    log = Log(os.path.join(args.out, "log"), out=sys.stderr)
+
+    def parse_polygon(feature_map, polygon, i):
+
+        try:
+            for i, ring in enumerate(polygon["coordinates"]):  # GeoJSON coordinates could be N dimensionals
+                polygon["coordinates"][i] = [[x, y] for point in ring for x, y in zip([point[0]], [point[1]])]
+
+            for tile in burntiles.burn([{"type": "feature", "geometry": polygon}], zoom=args.zoom):
+                feature_map[mercantile.Tile(*tile)].append({"type": "feature", "geometry": polygon})
+
+        except ValueError as e:
+            log.log("Warning: invalid feature {}, skipping".format(i))
+
+        return feature_map
+
+    def parse_geometry(feature_map, geometry, i):
+
+        if geometry["type"] == "Polygon":
+            feature_map = parse_polygon(feature_map, geometry, i)
+
+        elif geometry["type"] == "MultiPolygon":
+            for polygon in geometry["coordinates"]:
+                feature_map = parse_polygon(feature_map, {"type": "Polygon", "coordinates": polygon}, i)
+        else:
+            log.log("Notice: {} is a non surfacic geometry type, skipping feature {}".format(geometry["type"], i))
+
+        return feature_map
+
     for i, feature in enumerate(tqdm(fc["features"], ascii=True, unit="feature")):
 
-        if feature["geometry"]["type"] == "Polygon":
-            feature["geometry"]["coordinates"] = [feature["geometry"]["coordinates"]]
-            feature["geometry"]["type"] = "MultiPolygon"
-
-        for polygon in shape(feature["geometry"]):
-            simple_feature = {"type": "feature", "geometry": mapping(polygon)}
-            try:
-                for tile in burntiles.burn([simple_feature], zoom=args.zoom):
-                    feature_map[mercantile.Tile(*tile)].append(simple_feature)
-            except ValueError as e:
-                print("Warning: invalid feature {}, skipping".format(i), file=sys.stderr)
-                continue
+        if feature["geometry"]["type"] == "GeometryCollection":
+            for geometry in feature["geometry"]["geometries"]:
+                feature_map = parse_geometry(feature_map, geometry, i)
+        else:
+            feature_map = parse_geometry(feature_map, feature["geometry"], i)
 
     # Burn features to tiles and write to a slippy map directory.
     for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit="tile"):

From 4e795e070aac2973f71cd17009009b93c5864698 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 13 Nov 2018 23:37:53 +0100
Subject: [PATCH 54/97] Fix leaflet tiles parameter. Add comment on geojson
 input projection

---
 robosat/tools/rasterize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 32dbc8b8..d7cd19f7 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -47,6 +47,7 @@ def feature_to_mercator(feature):
     """
     # Ref: https://gist.github.com/dnomadb/5cbc116aacc352c7126e779c29ab7abe
 
+    # FIXME: We assume that GeoJSON input coordinates can't be anything else than EPSG:4326
     if feature["geometry"]["type"] == "Polygon":
         xys = (zip(*ring) for ring in feature["geometry"]["coordinates"])
         xys = (list(zip(*transform(CRS.from_epsg(4326), CRS.from_epsg(3857), *xy))) for xy in xys)
@@ -150,4 +151,4 @@ def parse_geometry(feature_map, geometry, i):
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles_from_csv(args.tiles), "png")
+        leaflet(args.out, args.leaflet, [tile for tile in tiles_from_csv(args.tiles)], "png")

From 23b9b96b4e93e01101affca1337c0ae5fbb17623 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 14 Nov 2018 18:10:51 +0100
Subject: [PATCH 55/97] Black is Black...

---
 robosat/colors.py | 2 +-
 robosat/tiles.py  | 2 +-
 robosat/utils.py  | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/robosat/colors.py b/robosat/colors.py
index fafbaa53..3996157e 100644
--- a/robosat/colors.py
+++ b/robosat/colors.py
@@ -99,7 +99,7 @@ def continuous_palette_for_color(color, bins=256):
 def complementary_palette(palette):
 
     comp_palette = []
-    colors = [palette[i:i+3] for i in range(0, len(palette), 3)]
+    colors = [palette[i : i + 3] for i in range(0, len(palette), 3)]
 
     for color in colors:
         r, g, b = [v for v in color]
diff --git a/robosat/tiles.py b/robosat/tiles.py
index c2391321..10539f6c 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -161,7 +161,7 @@ def adjacent_tile_image(tile, dx, dy, tiles):
     except KeyError:
         return None
 
-    assert path[-5:] == ".webp" # OpenCV AFAIK not handling PNG Palette
+    assert path[-5:] == ".webp"  # OpenCV AFAIK not handling PNG Palette
     return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
 
 
diff --git a/robosat/utils.py b/robosat/utils.py
index 78c54a15..0d8e2a78 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -1,5 +1,5 @@
 import re
-import os 
+import os
 from robosat.tiles import pixel_to_location
 import mercantile
 import json
@@ -37,7 +37,7 @@ def leaflet(out, base_url, tiles, ext):
     leaflet = re.sub("{{grid}}", grid, leaflet)
 
     # Could surely be improve, but for now, took the first tile to center on
-    tile = (list(tiles)[0])
+    tile = list(tiles)[0]
     x, y, z = map(int, [tile.x, tile.y, tile.z])
     leaflet = re.sub("{{zoom}}", str(z), leaflet)
     leaflet = re.sub("{{center}}", str(list(pixel_to_location(tile, 0.5, 0.5))[::-1]), leaflet)

From ce12ab5b61905c19e03dbc429f5b5b801156f7b0 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 14 Nov 2018 18:11:39 +0100
Subject: [PATCH 56/97] auto promote

---
 AUTHORS.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/AUTHORS.md b/AUTHORS.md
index ea00192d..6e935a78 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -1,3 +1,5 @@
 Daniel J. Hofmann <Hofmann@Mapbox.com> https://github.com/daniel-j-h
 
 Bhargav Kowshik <Bhargav@Mapbox.com> https://github.com/bkowshik
+
+Olivier Courtin <o@datapink.com> https://github.com/ocourtin

From 9ce178207127d0c465238f3206f723b00ddfb20d Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 14 Nov 2018 22:00:13 +0100
Subject: [PATCH 57/97] Switch default minimum to 0. Far more user-friendly

---
 robosat/tools/compare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 118f782e..82c8221e 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -29,7 +29,7 @@ def add_parser(subparser):
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, (for diff and list modes)")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
     parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip if foreground ratio below, [0-100]")
-    parser.add_argument("--minimum_qod", type=float, default=100.0, help="redshift tile if QoD below, [0-100]")
+    parser.add_argument("--minimum_qod", type=float, default=0.0, help="redshift tile if QoD below, [0-100]")
 
     parser.set_defaults(func=main)
 

From 3218253db0fbd4136796b811abb8da41fdf2d968 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Thu, 15 Nov 2018 07:58:19 +0100
Subject: [PATCH 58/97] Allow to specify which tile to grid on the leaflet
 client. Far more user friendly for compare diff use case. And so remove the
 red shift  tiling

---
 robosat/tools/compare.py   | 5 +++--
 robosat/tools/download.py  | 2 +-
 robosat/tools/masks.py     | 2 +-
 robosat/tools/predict.py   | 3 ++-
 robosat/tools/rasterize.py | 3 ++-
 robosat/tools/subset.py    | 2 +-
 robosat/tools/tile.py      | 3 ++-
 robosat/utils.py           | 4 ++--
 8 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 82c8221e..f33367b7 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -93,6 +93,7 @@ def main(args):
         palette_label = complementary_palette(palette_mask)
 
         images = tiles_from_slippy_map(args.images)
+        grid = []
 
         for tile, path in tqdm(list(images), desc="Compare", unit="image", ascii=True):
             x, y, z = list(map(str, tile))
@@ -109,7 +110,7 @@ def main(args):
 
             image = np.array(image) * 0.7
             if args.minimum_fg < fg_ratio and qod < args.minimum_qod:
-                image += np.array(Image.new("RGB", label.size, (int(255 * 0.3), 0, 0)))
+                grid.append(tile)
 
             mask.putpalette(palette_mask)
             label.putpalette(palette_label)
@@ -121,7 +122,7 @@ def main(args):
 
         if args.leaflet:
             tiles = [tile for tile, _ in tiles_from_slippy_map(args.images)]
-            leaflet(args.out, args.leaflet, tiles, "webp")
+            leaflet(args.out, args.leaflet, tiles, grid, "webp")
 
     elif args.mode == "list":
         if not args.labels or not args.masks or not args.dataset:
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 53995e04..945f6047 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -103,4 +103,4 @@ def worker(tile):
         log.log("Notice:\n {} tiles already downloads previously, so skipped now.".format(already_dl))
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles, args.ext)
+        leaflet(args.out, args.leaflet, tiles, tiles, args.ext)
diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index 7d72501a..25d906b5 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -72,7 +72,7 @@ def load(path):
 
     if args.leaflet:
         tiles = [tile for tile, _ in list(tiles_from_slippy_map(args.probs[0]))]
-        leaflet(args.masks, args.leaflet, tiles, "png")
+        leaflet(args.masks, args.leaflet, tiles, tiles, "png")
 
 
 def softvote(probs, axis=0, weights=None):
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 446141f1..abd58c30 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -109,4 +109,5 @@ def map_location(storage, _):
                 out.save(path, optimize=True)
 
     if args.leaflet:
-        leaflet(args.probs, args.leaflet, [tile for tile, _ in tiles_from_slippy_map(args.tiles)], "png")
+        tiles = [tile for tile, _ in tiles_from_slippy_map(args.tiles)]
+        leaflet(args.probs, args.leaflet, tiles, tiles, "png")
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index d7cd19f7..0e7341b6 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -151,4 +151,5 @@ def parse_geometry(feature_map, geometry, i):
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, [tile for tile in tiles_from_csv(args.tiles)], "png")
+        tiles = [tile for tile in tiles_from_csv(args.tiles)]
+        leaflet(args.out, args.leaflet, tiles, tiles, "png")
diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index a4d44ed9..015e7c1b 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -39,4 +39,4 @@ def main(args):
         shutil.copyfile(src, dst)
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles, extension)
+        leaflet(args.out, args.leaflet, tiles, tiles, extension)
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index b9284966..5cb7f587 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -119,4 +119,5 @@ def main(args):
             sys.exit("Error: Unknown type, should be either 'image' or 'label'")
 
     if args.leaflet:
-        leaflet(args.out, args.leaflet, [tile for tile in tiles if tile not in tiles_nodata], ext)
+        tiles = [tile for tile in tiles if tile not in tiles_nodata]
+        leaflet(args.out, args.leaflet, tiles, tiles, ext)
diff --git a/robosat/utils.py b/robosat/utils.py
index 0d8e2a78..7bbf28e2 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -27,9 +27,9 @@ def plot(out, history):
     plt.close()
 
 
-def leaflet(out, base_url, tiles, ext):
+def leaflet(out, base_url, tiles, grid, ext):
 
-    grid = json.dumps([mercantile.feature(tile) for tile in tiles])
+    grid = json.dumps([mercantile.feature(tile) for tile in grid]) if grid else "''"
 
     leaflet = open("./robosat/tools/templates/leaflet.html", "r").read()
     leaflet = re.sub("{{base_url}}", base_url, leaflet)

From f79fa97dddd831849e64299d20df9aba23588ece Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 16 Nov 2018 09:49:58 +0100
Subject: [PATCH 59/97] Prevent to delete an existing output directory while
 training. User friendly mandatory ^^

---
 robosat/tools/train.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 9e3bec79..7d1f7ebf 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -55,9 +55,15 @@ def add_parser(subparser):
 def main(args):
     model = load_config(args.model)
     dataset = load_config(args.dataset)
+    output_dir = model["common"]["checkpoint"]
 
-    os.makedirs(model["common"]["checkpoint"], exist_ok=True)
-    log = Log(os.path.join(model["common"]["checkpoint"], "log"))
+    if not args.resume:
+        try:
+            os.makedirs(output_dir)
+        except:
+            sys.exit("Can't create {} output dir, maybe because it's already exists ?".format(output_dir))
+
+    log = Log(os.path.join(output_dir, "log"))
 
     if torch.cuda.is_available():
         device = torch.device("cuda")
@@ -165,11 +171,11 @@ def map_location(storage, _):
             history["val " + k].append(v)
 
         visual = "history-{:05d}-of-{:05d}.png".format(epoch + 1, num_epochs)
-        plot(os.path.join(model["common"]["checkpoint"], visual), history)
+        plot(os.path.join(output_dir, visual), history)
 
         checkpoint = "checkpoint-{:05d}-of-{:05d}.pth".format(epoch + 1, num_epochs)
         states = {"epoch": epoch + 1, "state_dict": net.state_dict(), "optimizer": optimizer.state_dict()}
-        torch.save(states, os.path.join(model["common"]["checkpoint"], checkpoint))
+        torch.save(states, os.path.join(output_dir, checkpoint))
 
 
 def train(loader, num_classes, device, net, optimizer, criterion):

From b20561d61e837068945b199cbb4b0e3a448fef28 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 16 Nov 2018 15:23:42 +0100
Subject: [PATCH 60/97] Be more explicit if yes or no coverage is fully
 downloaded.

---
 robosat/tools/download.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 945f6047..4d1d0b5b 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -35,6 +35,7 @@ def add_parser(subparser):
 def main(args):
     tiles = list(tiles_from_csv(args.tiles))
     already_dl = 0
+    dl = 0
 
     with requests.Session() as session:
         num_workers = args.rate
@@ -94,13 +95,17 @@ def worker(tile):
                 return tile, url, True
 
             for tile, url, ok in executor.map(worker, tiles):
-                if not url and ok:
+                if url and ok:
+                    dl += 1
+                elif not url and ok:
                     already_dl += 1
-                if not ok:
+                else:
                     log.log("Warning:\n {} failed, skipping.\n {}\n".format(tile, url))
 
     if already_dl:
-        log.log("Notice:\n {} tiles already downloads previously, so skipped now.".format(already_dl))
+        log.log("Notice:\n {} tiles were already downloaded previously, and so skipped now.".format(already_dl))
+    if already_dl + dl == len(tiles):
+        log.log(" Coverage is fully downloaded.")
 
     if args.leaflet:
         leaflet(args.out, args.leaflet, tiles, tiles, args.ext)

From ccc7e8dd615e8e841cb53e73fe1fa5aa7e77a785 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 16 Nov 2018 17:09:16 +0100
Subject: [PATCH 61/97] not needed anymore

---
 robosat/unet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/robosat/unet.py b/robosat/unet.py
index ba50aadf..4f9fd799 100644
--- a/robosat/unet.py
+++ b/robosat/unet.py
@@ -12,7 +12,6 @@
 import torch
 import torch.nn as nn
 
-from copy import deepcopy
 from torchvision.models import resnet50
 
 

From 16a985dd8f903d1d7c8030658fee7a8309171a7f Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 16 Nov 2018 17:12:02 +0100
Subject: [PATCH 62/97] As types are not yet implemented no reason to ask yet
 for ^^

---
 config/dataset-parking.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/dataset-parking.toml b/config/dataset-parking.toml
index 1e5fd081..23148867 100644
--- a/config/dataset-parking.toml
+++ b/config/dataset-parking.toml
@@ -26,6 +26,5 @@
 # Channels configuration let your indicate wich dataset sub-directory and bands to take as input
 # You could so, add several channels blocks to compose your input Tensor. Orders are meaningful.
 [[channels]]
-type = "file"
 sub = "images"
 bands = [1,2,3]

From bc82113cccb727cef4993e1aebab2b626dd0ff5d Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:53:18 +0100
Subject: [PATCH 63/97]  Handle all CSS3 colors or hex pattern #RRGGBB. Use
 webcolors package

---
 config/dataset-parking.toml |  2 +-
 deps/requirements-lock.txt  |  1 +
 deps/requirements.txt       |  1 +
 robosat/colors.py           | 64 ++++++-------------------------------
 robosat/tools/compare.py    |  4 +--
 5 files changed, 14 insertions(+), 58 deletions(-)

diff --git a/config/dataset-parking.toml b/config/dataset-parking.toml
index 23148867..9e7065eb 100644
--- a/config/dataset-parking.toml
+++ b/config/dataset-parking.toml
@@ -12,7 +12,7 @@
   classes = ['background', 'parking']
 
   # Color map for visualization and representing classes in masks.
-  # Note: available colors can be found in `robosat/colors.py`
+  # Note: available colors are either CSS3 colors names or #RRGGBB hexadecimal representation.
   colors  = ['denim', 'orange']
 
 
diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 4d1b06d3..4ac56886 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -42,3 +42,4 @@ torchvision==0.2.1
 tqdm==4.23.4
 urllib3==1.22
 Werkzeug==0.14.1
+webcolors==1.8.1
diff --git a/deps/requirements.txt b/deps/requirements.txt
index a30503f8..19a5abe7 100644
--- a/deps/requirements.txt
+++ b/deps/requirements.txt
@@ -17,3 +17,4 @@ rtree
 pyproj
 toml
 pytest
+webcolors
diff --git a/robosat/colors.py b/robosat/colors.py
index 3996157e..3687e7b8 100644
--- a/robosat/colors.py
+++ b/robosat/colors.py
@@ -2,78 +2,30 @@
 """
 
 import colorsys
+import webcolors
 import numpy as np
 
-from enum import Enum, unique
-
-
-# Todo: user should be able to bring her own color palette.
-# Functions need to account for that and not use one palette.
-
-
-def _rgb(v):
-    r, g, b = v[1:3], v[3:5], v[5:7]
-    return int(r, 16), int(g, 16), int(b, 16)
-
-
-@unique
-class Mapbox(Enum):
-    """Mapbox-themed colors.
-
-    See: https://www.mapbox.com/base/styling/color/
-    """
-
-    dark = _rgb("#404040")
-    gray = _rgb("#eeeeee")
-    light = _rgb("#f8f8f8")
-    white = _rgb("#ffffff")
-    cyan = _rgb("#3bb2d0")
-    blue = _rgb("#3887be")
-    bluedark = _rgb("#223b53")
-    denim = _rgb("#50667f")
-    navy = _rgb("#28353d")
-    navydark = _rgb("#222b30")
-    purple = _rgb("#8a8acb")
-    teal = _rgb("#41afa5")
-    green = _rgb("#56b881")
-    yellow = _rgb("#f1f075")
-    mustard = _rgb("#fbb03b")
-    orange = _rgb("#f9886c")
-    red = _rgb("#e55e5e")
-    pink = _rgb("#ed6498")
-
 
 def make_palette(*colors):
-    """Builds a PIL-compatible color palette from color names.
+    """Builds a PIL-compatible color palette from CSS3 color names, or hex values patterns as #RRGGBB
 
     Args:
       colors: variable number of color names.
     """
 
     assert 0 < len(colors) <= 256
-    rgbs = [Mapbox[color].value for color in colors]
 
-    return list(sum(rgbs, ()))
-
-
-def color_string_to_rgb(color):
-    """Convert color string to a list of RBG integers.
-
-    Args:
-      color: the string color value for example "250,0,0"
+    hexs = [webcolors.CSS3_NAMES_TO_HEX[color] if color[0] != "#" else color for color in colors]
+    rgbs = [(int(h[1:3], 16), int(h[3:5], 16), int(h[5:7], 16)) for h in hexs]
 
-    Returns:
-      color: as a list of RGB integers for example [250,0,0]
-    """
-
-    return [*map(int, color.split(","))]
+    return list(sum(rgbs, ()))
 
 
 def continuous_palette_for_color(color, bins=256):
     """Creates a continuous color palette based on a single color.
 
     Args:
-      color: the rgb color tuple to create a continuous palette for.
+      color: the CSS3 color name or it's hex values as #RRGGBB, to create a continuous palette for.
       bins: the number of colors to create in the continuous palette.
 
     Returns:
@@ -83,7 +35,8 @@ def continuous_palette_for_color(color, bins=256):
     # A quick and dirty way to create a continuous color palette is to convert from the RGB color
     # space into the HSV color space and then only adapt the color's saturation (S component).
 
-    r, g, b = [v / 255 for v in Mapbox[color].value]
+    hexs = webcolors.CSS3_NAMES_TO_HEX[color] if color[0] != "#" else color
+    r, g, b = [(int(h[1:3], 16), int(h[3:5], 16), int(h[5:7], 16)) for h in hexs]
     h, s, v = colorsys.rgb_to_hsv(r, g, b)
 
     assert 0 < bins <= 256
@@ -97,6 +50,7 @@ def continuous_palette_for_color(color, bins=256):
 
 
 def complementary_palette(palette):
+    """Creates a PIL complementary colors palette based on an initial PIL palette"""
 
     comp_palette = []
     colors = [palette[i : i + 3] for i in range(0, len(palette), 3)]
diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index f33367b7..0d50ba5f 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -28,8 +28,8 @@ def add_parser(subparser):
     parser.add_argument("--mode", type=str, default="side", help="compare mode (e.g side, diff or list)")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, (for diff and list modes)")
     parser.add_argument("--leaflet", type=str, help="leaflet client base url")
-    parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip if foreground ratio below, [0-100]")
-    parser.add_argument("--minimum_qod", type=float, default=0.0, help="redshift tile if QoD below, [0-100]")
+    parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip tile if foreground ratio below, [0-100]")
+    parser.add_argument("--minimum_qod", type=float, default=0.0, help="select tile if QoD below, [0-100]")
 
     parser.set_defaults(func=main)
 

From 008398efe44a9795ae8a3c6c5498a46709132a7f Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:54:57 +0100
Subject: [PATCH 64/97] Remove deprecated train checkpoint dir. Add forgotten
 pretrained parameter

---
 config/model-unet.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/model-unet.toml b/config/model-unet.toml
index 5c4f3c20..802f17f3 100644
--- a/config/model-unet.toml
+++ b/config/model-unet.toml
@@ -11,9 +11,6 @@
   # Image side size in pixels.
   image_size = 512
 
-  # Directory where to save checkpoints to during training.
-  checkpoint = '/tmp/pth/'
-
 
 # Model specific optimization parameters.
 [opt]
@@ -32,3 +29,6 @@
 
   # Data augmentation, Flip or Rotate probability
   data_augmentation = 0.75
+
+  # Use ImageNet weights pretraining
+  pretrained = true

From 583f1ceb7e38a4cb946b85794b3fac473eba186a Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:55:21 +0100
Subject: [PATCH 65/97] Improve log resilience. Add train out parameter
 mandatory. web_ui switch

---
 robosat/log.py         | 28 +++++++++++++++++++---------
 robosat/tools/train.py | 20 +++++++-------------
 2 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/robosat/log.py b/robosat/log.py
index e59e3b50..9f2ed9ae 100644
--- a/robosat/log.py
+++ b/robosat/log.py
@@ -10,18 +10,28 @@ class Log:
     """Create a log instance on a log file
     """
 
-    def __init__(self, path, out=sys.stdout, mode="a"):
+    def __init__(self, path, out=None):
+
+        self.fp = None
         self.out = out
-        self.fp = open(path, mode)
-        assert self.fp, "Unable to open log file"
+        try:
+            if path:
+                if not os.path.isdir(os.path.dirname(path)):
+                    os.makedirs(os.path.dirname(path), exist_ok=True)
+                self.fp = open(path, mode="a")
+        except:
+            sys.exit("Unable to write in log directory")
 
     """Log a new message to the opened log file, and optionnaly on stdout or stderr too
     """
 
     def log(self, msg):
-        assert self.fp, "Unable to write in log file"
-        self.fp.write(msg + os.linesep)
-        self.fp.flush()
-
-        if self.out:
-            print(msg, file=self.out)
+        try:
+            if self.fp:
+                self.fp.write(msg + os.linesep)
+                self.fp.flush()
+
+            if self.out:
+                print(msg, file=self.out)
+        except:
+            sys.exit("Unable to write in log file")
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 7d1f7ebf..a5ec4ae4 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -43,6 +43,7 @@ def add_parser(subparser):
         "train", help="trains model on dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
+    parser.add_argument("out", type=str, help="directory to save checkpoint .pth files and log")
     parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--checkpoint", type=str, required=False, help="path to a model checkpoint (to retrain)")
@@ -55,15 +56,8 @@ def add_parser(subparser):
 def main(args):
     model = load_config(args.model)
     dataset = load_config(args.dataset)
-    output_dir = model["common"]["checkpoint"]
 
-    if not args.resume:
-        try:
-            os.makedirs(output_dir)
-        except:
-            sys.exit("Can't create {} output dir, maybe because it's already exists ?".format(output_dir))
-
-    log = Log(os.path.join(output_dir, "log"))
+    log = Log(os.path.join(args.out, "log"))
 
     if torch.cuda.is_available():
         device = torch.device("cuda")
@@ -143,6 +137,7 @@ def map_location(storage, _):
     log.log("")
 
     for epoch in range(resume, num_epochs):
+
         log.log("---")
         log.log("Epoch: {}/{}".format(epoch + 1, num_epochs))
 
@@ -169,13 +164,12 @@ def map_location(storage, _):
 
         for k, v in val_hist.items():
             history["val " + k].append(v)
+        visual_path = os.path.join(args.out, "history-{:05d}-of-{:05d}.png".format(epoch + 1, num_epochs))
+        plot(visual_path, history)
 
-        visual = "history-{:05d}-of-{:05d}.png".format(epoch + 1, num_epochs)
-        plot(os.path.join(output_dir, visual), history)
-
-        checkpoint = "checkpoint-{:05d}-of-{:05d}.pth".format(epoch + 1, num_epochs)
         states = {"epoch": epoch + 1, "state_dict": net.state_dict(), "optimizer": optimizer.state_dict()}
-        torch.save(states, os.path.join(output_dir, checkpoint))
+        checkpoint_path = os.path.join(args.out, "checkpoint-{:05d}-of-{:05d}.pth".format(epoch + 1, num_epochs))
+        torch.save(states, checkpoint_path)
 
 
 def train(loader, num_classes, device, net, optimizer, criterion):

From 47d91a4a335a5c389744e72cdefd4ecf80c58f59 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:56:00 +0100
Subject: [PATCH 66/97] new tile_image function

---
 robosat/tiles.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index 10539f6c..c2a38cba 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -11,8 +11,10 @@
 import csv
 import io
 import os
+from glob import glob
 
 import cv2
+from PIL import Image
 import numpy as np
 
 from rasterio.warp import transform
@@ -140,6 +142,19 @@ def tiles_from_csv(path):
             yield mercantile.Tile(*map(int, row))
 
 
+def tile_image(root, x, y, z):
+    """Retrieves H,W,C numpy array, from a tile store and X,Y,Z coordinates, or `None`"""
+
+    try:
+        path = glob(os.path.join(root, z, x, y) + "*")
+        assert len(path) == 1
+        img = np.array(Image.open(path[0]).convert("RGB"))
+    except:
+        return None
+
+    return img
+
+
 def adjacent_tile_image(tile, dx, dy, tiles):
     """Retrieves an adjacent tile image from a tile store.
 
@@ -161,7 +176,6 @@ def adjacent_tile_image(tile, dx, dy, tiles):
     except KeyError:
         return None
 
-    assert path[-5:] == ".webp"  # OpenCV AFAIK not handling PNG Palette
     return cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB)
 
 
@@ -175,7 +189,7 @@ def buffer_tile_image(tile, tiles, overlap, tile_size):
       tile_size: the tile size.
 
     Returns:
-      The composite image containing the original tile plus tile overlap on all sides.
+      The H,W,C numpy composite image containing the original tile plus tile overlap on all sides.
       It's size is `tile_size` + 2 * `overlap` pixel for each side.
     """
 

From 293673e5be820b905224e0bd85fe8fb0894f5dff Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:56:27 +0100
Subject: [PATCH 67/97] Whole refactor, on my previous whole refactor ^^

---
 robosat/tools/compare.py | 218 +++++++++++++++++++++------------------
 1 file changed, 119 insertions(+), 99 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 0d50ba5f..8af7468e 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import math
+import json
 import torch
 import argparse
 
@@ -8,35 +9,46 @@
 from tqdm import tqdm
 import numpy as np
 
+from mercantile import feature
+
 from robosat.colors import make_palette, complementary_palette
-from robosat.tiles import tiles_from_slippy_map
+from robosat.tiles import tiles_from_slippy_map, tile_image
 from robosat.config import load_config
 from robosat.metrics import Metrics
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 from robosat.log import Log
 
 
 def add_parser(subparser):
     parser = subparser.add_parser(
-        "compare", help="compare images, labels and masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        "compare", help="compare images and/or labels and masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    parser.add_argument("out", type=str, help="directory to save output to (or path in list mode)")
-    parser.add_argument("--images", type=str, help="directory to read slippy map images from")
-    parser.add_argument("--labels", type=str, help="directory to read slippy map labels from")
-    parser.add_argument("--masks", type=str, help="directory to read slippy map masks from")
-    parser.add_argument("--dirs", type=str, nargs="+", help="slippy map directories to compares (in side mode)")
-    parser.add_argument("--mode", type=str, default="side", help="compare mode (e.g side, diff or list)")
-    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, (for diff and list modes)")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
-    parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip tile if foreground ratio below, [0-100]")
-    parser.add_argument("--minimum_qod", type=float, default=0.0, help="select tile if QoD below, [0-100]")
+    parser.add_argument("--mode", type=str, default="side", help="compare mode (e.g side, stack or list)")
+    parser.add_argument("--images", type=str, nargs="+", help="slippy map images dirs to render (stack or side mode)")
+    parser.add_argument("--ext", type=str, default="webp", help="file format to save images in (stack or side mode)")
+    parser.add_argument("--labels", type=str, help="directory to read slippy map labels from (needed for QoD metric)")
+    parser.add_argument("--masks", type=str, help="directory to read slippy map masks from (needed for QoD metric)")
+    parser.add_argument("--dataset", type=str, help="path to dataset configuration file (needed for QoD metric)")
+    parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip tile if label foreground below, [0-100]")
+    parser.add_argument("--maximum_fg", type=float, default=100.0, help="skip tile if label foreground above, [0-100]")
+    parser.add_argument("--minimum_qod", type=float, default=0.0, help="skip tile if QoD metric below, [0-100]")
+    parser.add_argument("--maximum_qod", type=float, default=100.0, help="skip tile if QoD metric above, [0-100]")
+    parser.add_argument("--vertical", action="store_true", help="render vertical image aggregate, for side mode")
+    parser.add_argument("--geojson", action="store_true", help="output geojson based, for list mode")
+    parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("out", type=str, help="directory or path (upon mode) to save output to")
 
     parser.set_defaults(func=main)
 
 
-def compare(mask, label, classes):
+def compare(masks, labels, tile, classes):
+
+    x, y, z = list(map(str, tile))
+    label = np.array(Image.open(os.path.join(labels, z, x, "{}.png".format(y))))
+    mask = np.array(Image.open(os.path.join(masks, z, x, "{}.png".format(y))))
 
-    # TODO: Still binary class centric
+    assert label.shape == mask.shape
+    assert len(label.shape) == 2 and len(classes) == 2  # Still binary centric
 
     metrics = Metrics(classes)
     metrics.add(torch.from_numpy(label), torch.from_numpy(mask), is_prob=False)
@@ -53,94 +65,102 @@ def compare(mask, label, classes):
 
 def main(args):
 
-    if args.mode == "side":
-        if not args.dirs or len(args.dirs) < 2:
-            sys.exit("Error: In side mode, you must provide at least two directories")
-
-        tiles = tiles_from_slippy_map(args.dirs[0])
-
-        for tile, path in tqdm(list(tiles), desc="Compare", unit="image", ascii=True):
-
-            width, heigh = Image.open(path).size
-            side = Image.new(mode="RGB", size=(len(args.dirs) * width, height))
-
-            x, y, z = list(map(str, tile))
-
-            for i, dir in enumerate(args.dirs):
-                try:
-                    img = Image.open(os.path.join(dir, z, x, "{}.png".format(y)))
-                except:
-                    img = Image.open(os.path.join(dir, z, x, "{}.webp".format(y))).convert("RGB")
-
-                assert image.size == img.size
-                side.paste(img, box=(i * width, 0))
+    if not args.masks or not args.labels or not args.dataset:
+        if args.mode == "list":
+            sys.exit("Parameters masks, labels and dataset, are all mandatories in list mode.")
+        if args.minimum_fg > 0 or args.maximum_fg < 100 or args.minimum_qod > 0 or args.maximum_qod < 100:
+            sys.exit("Parameters masks, labels and dataset, are all mandatories in QoD filtering.")
+
+    if args.images:
+        tiles = [tile for tile, _ in tiles_from_slippy_map(args.images[0])]
+        for image in args.images[1:]:
+            assert sorted(tiles) == sorted([tile for tile, _ in tiles_from_slippy_map(image)]), "inconsistent coverages"
+
+    if args.labels and args.masks:
+        tiles_masks = [tile for tile, _ in tiles_from_slippy_map(args.masks)]
+        tiles_labels = [tile for tile, _ in tiles_from_slippy_map(args.labels)]
+        if args.images:
+            assert sorted(tiles) == sorted(tiles_masks) == sorted(tiles_labels), "inconsistent coverages"
+        else:
+            assert sorted(tiles_masks) == sorted(tiles_labels), "inconsistent coverages"
+            tiles = tiles_masks
+
+    if args.mode == "list":
+        out = open(args.out, mode="w")
+        if args.geojson:
+            out.write('{"type":"FeatureCollection","features":[')
+            first = True
+
+    tiles_compare = []
+    for tile in tqdm(list(tiles), desc="Compare", unit="tile", ascii=True):
+
+        x, y, z = list(map(str, tile))
+
+        if args.masks and args.labels and args.dataset:
+            classes = load_config(args.dataset)["common"]["classes"]
+            dist, fg_ratio, qod = compare(args.masks, args.labels, tile, classes)
+            if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod:
+                continue
+
+        tiles_compare.append(tile)
+
+        if args.mode == "side":
+
+            for i, image in enumerate(args.images):
+                img = tile_image(image, x, y, z)
+
+                if i == 0:
+                    side = np.zeros((img.shape[0], img.shape[1] * len(args.images), 3))
+                    side = np.swapaxes(side, 0, 1) if args.vertical else side
+                    image_shape = img.shape
+                else:
+                    assert image_shape == img.shape, "Unconsistent image size to compare"
+
+                if args.vertical:
+                    side[i * image_shape[0] : (i + 1) * image_shape[0], :, :] = img
+                else:
+                    side[:, i * image_shape[0] : (i + 1) * image_shape[0], :] = img
 
             os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
-            path = os.path.join(args.out, z, x, "{}.webp".format(y))
-            side.save(path, optimize=True)
-
-    elif args.mode == "diff":
-        if not args.images or not args.labels or not args.masks or not args.dataset:
-            sys.exit("Error: in diff mode, you must provide images, labels and masks directories, and dataset path")
+            side = Image.fromarray(np.uint8(side))
+            side.save(os.path.join(args.out, z, x, "{}.{}".format(y, args.ext)), optimize=True)
 
-        dataset = load_config(args.dataset)
-        classes = dataset["common"]["classes"]
-        colors = dataset["common"]["colors"]
-        assert len(classes) == len(colors), "classes and colors coincide"
-        assert len(colors) == 2, "only binary models supported right now"
+        elif args.mode == "stack":
 
-        palette_mask = make_palette(colors[0], colors[1])
-        palette_label = complementary_palette(palette_mask)
+            for i, image in enumerate(args.images):
+                img = tile_image(image, x, y, z)
 
-        images = tiles_from_slippy_map(args.images)
-        grid = []
+                if i == 0:
+                    image_shape = img.shape[0:2]
+                    stack = img / len(args.images)
+                else:
+                    assert image_shape == img.shape[0:2], "Unconsistent image size to compare"
+                    stack = stack + (img / len(args.images))
 
-        for tile, path in tqdm(list(images), desc="Compare", unit="image", ascii=True):
-            x, y, z = list(map(str, tile))
             os.makedirs(os.path.join(args.out, str(z), str(x)), exist_ok=True)
-
-            image = Image.open(path).convert("RGB")
-            label = Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))
-            mask = Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))
-
-            assert image.size == label.size == mask.size
-            assert label.getbands() == mask.getbands() == tuple("P")
-
-            dist, fg_ratio, qod = compare(np.array(mask), np.array(label), classes)
-
-            image = np.array(image) * 0.7
-            if args.minimum_fg < fg_ratio and qod < args.minimum_qod:
-                grid.append(tile)
-
-            mask.putpalette(palette_mask)
-            label.putpalette(palette_label)
-            mask = np.array(mask.convert("RGB"))
-            label = np.array(label.convert("RGB"))
-
-            diff = Image.fromarray(np.uint8((image + mask + label) / 3.0))
-            diff.save(os.path.join(args.out, str(z), str(x), "{}.webp".format(y)), optimize=True)
-
-        if args.leaflet:
-            tiles = [tile for tile, _ in tiles_from_slippy_map(args.images)]
-            leaflet(args.out, args.leaflet, tiles, grid, "webp")
-
-    elif args.mode == "list":
-        if not args.labels or not args.masks or not args.dataset:
-            sys.exit("In list mode, you must provide labels and masks directories, and dataset path")
-
-        dataset = load_config(args.dataset)
-        masks = tiles_from_slippy_map(args.masks)
-        os.makedirs(os.path.basename(args.out), exist_ok=True)
-        log = Log(args.out, out=None, mode="w")
-
-        for tile, path in tqdm(list(masks), desc="Compare", unit="image", ascii=True):
-            x, y, z = list(map(str, tile))
-            mask = Image.open(os.path.join(args.masks, z, x, "{}.png".format(y)))
-            label = Image.open(os.path.join(args.labels, z, x, "{}.png".format(y)))
-
-            assert label.size == mask.size
-            assert label.getbands() == mask.getbands() == tuple("P")
-
-            dist, fg_ratio, qod = compare(np.array(mask), np.array(label), dataset["common"]["classes"])
-            if args.minimum_fg < fg_ratio and qod < args.minimum_qod:
-                log.log("{},{},{}\t\t{:.3f}\t\t{:.3f}\t\t{:.3f}".format(x, y, z, dist, fg_ratio, qod))
+            stack = Image.fromarray(np.uint8(stack))
+            stack.save(os.path.join(args.out, str(z), str(x), "{}.{}".format(y, args.ext)), optimize=True)
+
+        elif args.mode == "list":
+            if args.geojson:
+                prop = '"properties":{{"x":{},"y":{},"z":{},"fg":{:.1f},"qod":{:.1f}}}'.format(x, y, z, fg_ratio, qod)
+                geom = '"geometry":{}'.format(json.dumps(feature(tile, precision=6)["geometry"]))
+                out.write('{}{{"type":"Feature",{},{}}}'.format("," if not first else "", geom, prop))
+                first = False
+            else:
+                out.write("{},{},{}\t\t{:.1f}\t\t{:.1f}{}".format(x, y, z, fg_ratio, qod, os.linesep))
+
+        else:
+            sys.exit("Unkown mode, should be either: side, stack or list")
+
+    if args.mode == "list":
+        if args.geojson:
+            out.write("]}")
+        out.close()
+
+    elif args.mode == "side" and args.web_ui:
+        web_ui(args.out, args.web_ui, None, tiles_compare, args.ext, "compare.html")
+
+    elif args.mode == "stack" and args.web_ui:
+        tiles = [tile for tile, _ in tiles_from_slippy_map(args.images[0])]
+        web_ui(args.out, args.web_ui, tiles, tiles_compare, args.ext, "leaflet.html")

From e3cfd3a86e9507030fe7a80f398f2d0fffa18b92 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:56:59 +0100
Subject: [PATCH 68/97] web_ui switch. Color process harmonization

---
 robosat/tools/masks.py     | 13 ++++++++-----
 robosat/tools/predict.py   | 10 +++++-----
 robosat/tools/rasterize.py | 15 ++++++---------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index 25d906b5..f9f12ecc 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -9,7 +9,7 @@
 
 from robosat.tiles import tiles_from_slippy_map
 from robosat.colors import make_palette
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 
 
 def add_parser(subparser):
@@ -21,8 +21,9 @@ def add_parser(subparser):
 
     parser.add_argument("masks", type=str, help="slippy map directory to save masks to")
     parser.add_argument("probs", type=str, nargs="+", help="slippy map directories with class probabilities")
+    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--weights", type=float, nargs="+", help="weights for weighted average soft-voting")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--web_ui", type=str, help="web ui client base url")
 
     parser.set_defaults(func=main)
 
@@ -61,7 +62,9 @@ def load(path):
         mask = softvote(probs, axis=0, weights=args.weights)
         mask = mask.astype(np.uint8)
 
-        palette = make_palette("denim", "orange")
+        dataset = load_config(args.dataset)
+        palette = make_palette(dataset["common"]["colors"][0], dataset["common"]["colors"][1])
+
         out = Image.fromarray(mask, mode="P")
         out.putpalette(palette)
 
@@ -70,9 +73,9 @@ def load(path):
         path = os.path.join(args.masks, str(z), str(x), str(y) + ".png")
         out.save(path, optimize=True)
 
-    if args.leaflet:
+    if args.web_ui:
         tiles = [tile for tile, _ in list(tiles_from_slippy_map(args.probs[0]))]
-        leaflet(args.masks, args.leaflet, tiles, tiles, "png")
+        web_ui(args.masks, args.web_ui, tiles, tiles, "png", "leaflet.html")
 
 
 def softvote(probs, axis=0, weights=None):
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index abd58c30..4ab5e078 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -19,7 +19,7 @@
 from robosat.config import load_config
 from robosat.colors import continuous_palette_for_color, make_palette
 from robosat.transforms import ImageToTensor
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 
 
 def add_parser(subparser):
@@ -38,7 +38,7 @@ def add_parser(subparser):
     parser.add_argument("probs", type=str, help="directory to save slippy map probability masks to")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--web_ui", type=str, help="web ui base url")
 
     parser.set_defaults(func=main)
 
@@ -73,7 +73,7 @@ def map_location(storage, _):
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
 
     if args.masks_output:
-        palette = make_palette("white", "pink")
+        palette = make_palette(dataset["common"]["colors"][0], dataset["common"]["colors"][1])
     else:
         palette = continuous_palette_for_color("pink", 256)
 
@@ -108,6 +108,6 @@ def map_location(storage, _):
 
                 out.save(path, optimize=True)
 
-    if args.leaflet:
+    if args.web_ui:
         tiles = [tile for tile, _ in tiles_from_slippy_map(args.tiles)]
-        leaflet(args.probs, args.leaflet, tiles, tiles, "png")
+        web_ui(args.probs, args.web_ui, tiles, tiles, "png", "leaflet.html")
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 0e7341b6..3d55375e 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -17,9 +17,9 @@
 from shapely.geometry import mapping
 
 from robosat.config import load_config
-from robosat.colors import make_palette
+from robosat.colors import make_palette, complementary_palette
 from robosat.tiles import tiles_from_csv
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 from robosat.log import Log
 
 
@@ -34,7 +34,7 @@ def add_parser(subparser):
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--size", type=int, default=512, help="size of rasterized image tiles in pixels")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--web_ui", type=str, help="web ui client base url")
 
     parser.set_defaults(func=main)
 
@@ -83,10 +83,7 @@ def main(args):
     classes = dataset["common"]["classes"]
     colors = dataset["common"]["colors"]
     assert len(classes) == len(colors), "classes and colors coincide"
-
     assert len(colors) == 2, "only binary models supported right now"
-    bg = colors[0]
-    fg = colors[1]
 
     os.makedirs(args.out, exist_ok=True)
 
@@ -142,7 +139,7 @@ def parse_geometry(feature_map, geometry, i):
         else:
             out = Image.fromarray(np.zeros(shape=(args.size, args.size)).astype(int), mode="P")
 
-        palette = make_palette(bg, fg)
+        palette = complementary_palette(make_palette(colors[0], colors[1]))
         out.putpalette(palette)
 
         out_path = os.path.join(args.out, str(tile.z), str(tile.x))
@@ -150,6 +147,6 @@ def parse_geometry(feature_map, geometry, i):
 
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
-    if args.leaflet:
+    if args.web_ui:
         tiles = [tile for tile in tiles_from_csv(args.tiles)]
-        leaflet(args.out, args.leaflet, tiles, tiles, "png")
+        web_ui(args.out, args.web_ui, tiles, tiles, "png", "leaflet.html")

From cce48b6fce6275c384cf01bf32101723dcb3badf Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:57:21 +0100
Subject: [PATCH 69/97] web_ui switch

---
 robosat/tools/download.py | 8 ++++----
 robosat/tools/tile.py     | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 4d1d0b5b..2a3a4f78 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -9,7 +9,7 @@
 from tqdm import tqdm
 
 from robosat.tiles import tiles_from_csv, fetch_image, tile_to_bbox
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 from robosat.log import Log
 
 
@@ -25,9 +25,9 @@ def add_parser(subparser):
     parser.add_argument("--rate", type=int, default=10, help="rate limit in max. requests per second")
     parser.add_argument("--type", type=str, default="XYZ", help="service type to use (e.g: XYZ, WMS or TMS)")
     parser.add_argument("--timeout", type=int, default=10, help="server request timeout (in seconds)")
+    parser.add_argument("--web_ui", type=str, help="web ui client base url")
     parser.add_argument("tiles", type=str, help="path to .csv tiles file")
     parser.add_argument("out", type=str, help="path to slippy map directory for storing tiles")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
 
     parser.set_defaults(func=main)
 
@@ -107,5 +107,5 @@ def worker(tile):
     if already_dl + dl == len(tiles):
         log.log(" Coverage is fully downloaded.")
 
-    if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles, tiles, args.ext)
+    if args.web_ui:
+        web_ui(args.out, args.web_ui, tiles, tiles, args.ext, "leaflet.html")
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 5cb7f587..1b3df199 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -17,7 +17,7 @@
 
 from robosat.config import load_config
 from robosat.colors import make_palette
-from robosat.utils import leaflet
+from robosat.utils import web_ui
 
 
 def add_parser(subparser):
@@ -32,7 +32,7 @@ def add_parser(subparser):
     parser.add_argument("--type", type=str, default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
     parser.add_argument("--no_data", type=int, help="color considered as no data [0-255]. Skip related tile")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--web_ui", type=str, help="web ui base url")
 
     parser.set_defaults(func=main)
 
@@ -118,6 +118,6 @@ def main(args):
         else:
             sys.exit("Error: Unknown type, should be either 'image' or 'label'")
 
-    if args.leaflet:
+    if args.web_ui:
         tiles = [tile for tile in tiles if tile not in tiles_nodata]
-        leaflet(args.out, args.leaflet, tiles, tiles, ext)
+        web_ui(args.out, args.web_ui, tiles, tiles, ext, "leaflet.html")

From 30a68adb8821eeae764faaa573335fca38f6fb0c Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:58:01 +0100
Subject: [PATCH 70/97] Use the cover filter as a filter (rather than parsing
 the whole tiles dir). Add move option. Add log support. web_ui switch

---
 robosat/tools/subset.py | 54 ++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index 015e7c1b..1c385b3a 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -1,11 +1,14 @@
 import os
+import sys
 import argparse
 import shutil
 
+from glob import glob
 from tqdm import tqdm
 
-from robosat.tiles import tiles_from_slippy_map, tiles_from_csv
-from robosat.utils import leaflet
+from robosat.tiles import tiles_from_csv
+from robosat.utils import web_ui
+from robosat.log import Log
 
 
 def add_parser(subparser):
@@ -14,29 +17,40 @@ def add_parser(subparser):
         help="filter images in a slippy map directory using a csv",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
-    parser.add_argument("images", type=str, help="directory to read slippy map image tiles from for filtering")
-    parser.add_argument("tiles", type=str, help="csv to filter images by")
-    parser.add_argument("out", type=str, help="directory to save filtered images to")
-    parser.add_argument("--leaflet", type=str, help="leaflet client base url")
+    parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("--move", action="store_true", help="move files from src to dst (rather than copy them)" )
+    parser.add_argument("dir", type=str, help="directory to read slippy map tiles from for filtering")
+    parser.add_argument("cover", type=str, help="csv cover to filter tiles by")
+    parser.add_argument("out", type=str, help="directory to save filtered tiles to")
 
     parser.set_defaults(func=main)
 
 
 def main(args):
-    images = tiles_from_slippy_map(args.images)
+    log = Log(os.path.join(args.out, "log"), out=sys.stderr)
 
-    tiles = set(tiles_from_csv(args.tiles))
+    tiles = set(tiles_from_csv(args.cover))
+    extension = ""
 
-    for tile, src in tqdm(list(images), desc="Subset", unit="image", ascii=True):
-        if tile not in tiles:
-            continue
-
-        extension = os.path.splitext(src)[1][1:]
-
-        os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
-        dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension))
+    for tile in tqdm(tiles, desc="Subset", unit="tiles", ascii=True):
 
-        shutil.copyfile(src, dst)
-
-    if args.leaflet:
-        leaflet(args.out, args.leaflet, tiles, tiles, extension)
+        paths = glob(os.path.join(args.dir, str(tile.z), str(tile.x), "{}.*".format(tile.y)))
+        if len(paths) != 1:
+            log.log("Warning: {} skipped.".format(tile))
+            continue
+        src = paths[0]
+
+        try:
+            extension = os.path.splitext(src)[1][1:]
+            dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension))
+            os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
+            if args.move:
+                assert(os.path.isfile(src))
+                shutil.move(src, dst)
+            else:
+                shutil.copyfile(src, dst)
+        except:
+            sys.exit("Error: Unable to process {}".format(tile))
+
+    if args.web_ui:
+        web_ui(args.out, args.web_ui, tiles, tiles, extension, "leaflet.html")

From 42c110bddfbe208d3d5c068e1fda2d1adca4707e Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:58:28 +0100
Subject: [PATCH 71/97] Improve GeoJSON tiles: add x,y,z properties,
 ligthweight coordinates precision, sava as an external file. web_ui switch

---
 robosat/utils.py | 46 ++++++++++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 16 deletions(-)

diff --git a/robosat/utils.py b/robosat/utils.py
index 7bbf28e2..52ae60f6 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -1,9 +1,9 @@
 import re
 import os
-from robosat.tiles import pixel_to_location
-import mercantile
 import json
 import matplotlib
+from mercantile import feature
+from robosat.tiles import pixel_to_location
 
 matplotlib.use("Agg")
 import matplotlib.pyplot as plt  # noqa: E402
@@ -27,21 +27,35 @@ def plot(out, history):
     plt.close()
 
 
-def leaflet(out, base_url, tiles, grid, ext):
+def web_ui(out, base_url, coverage_tiles, selected_tiles, ext, template):
+
+    try:
+        web_ui = open(os.path.join("./robosat/tools/templates/" + template), "r").read()
+    except:
+        sys.exit("Unable to open Web UI template {}".format(template))
+
+    web_ui = re.sub("{{base_url}}", base_url, web_ui)
+    web_ui = re.sub("{{ext}}", ext, web_ui)
+    web_ui = re.sub("{{tiles}}", "tiles.json" if selected_tiles else "''", web_ui)
 
-    grid = json.dumps([mercantile.feature(tile) for tile in grid]) if grid else "''"
+    if coverage_tiles:
+        # Could surely be improve, but for now, took the first tile to center on
+        tile = list(coverage_tiles)[0]
+        x, y, z = map(int, [tile.x, tile.y, tile.z])
+        web_ui = re.sub("{{zoom}}", str(z), web_ui)
+        web_ui = re.sub("{{center}}", str(list(pixel_to_location(tile, 0.5, 0.5))[::-1]), web_ui)
 
-    leaflet = open("./robosat/tools/templates/leaflet.html", "r").read()
-    leaflet = re.sub("{{base_url}}", base_url, leaflet)
-    leaflet = re.sub("{{ext}}", ext, leaflet)
-    leaflet = re.sub("{{grid}}", grid, leaflet)
+    with open(os.path.join(out, "index.html"), "w", encoding="utf-8") as fp:
+        fp.write(web_ui)
 
-    # Could surely be improve, but for now, took the first tile to center on
-    tile = list(tiles)[0]
-    x, y, z = map(int, [tile.x, tile.y, tile.z])
-    leaflet = re.sub("{{zoom}}", str(z), leaflet)
-    leaflet = re.sub("{{center}}", str(list(pixel_to_location(tile, 0.5, 0.5))[::-1]), leaflet)
+    if selected_tiles:
+        with open(os.path.join(out, "tiles.json"), "w", encoding="utf-8") as fp:
+            fp.write('{"type":"FeatureCollection","features":[')
+            first = True
+            for tile in selected_tiles:
+                prop = '"properties":{{"x":{},"y":{},"z":{}}}'.format(int(tile.x), int(tile.y), int(tile.z))
+                geom = '"geometry":{}'.format(json.dumps(feature(tile, precision=6)["geometry"]))
+                fp.write('{}{{"type":"Feature",{},{}}}'.format("," if not first else "", geom, prop))
+                first = False
+            fp.write("]}")
 
-    f = open(os.path.join(out, "index.html"), "w", encoding="utf-8")
-    f.write(leaflet)
-    f.close()

From 6ead578005658718e9c72d904f3586c724bf1e35 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:58:46 +0100
Subject: [PATCH 72/97] Use external GeoJSON grid file. Ajax loading

---
 robosat/tools/templates/leaflet.html | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/robosat/tools/templates/leaflet.html b/robosat/tools/templates/leaflet.html
index 47121176..4867ec0b 100644
--- a/robosat/tools/templates/leaflet.html
+++ b/robosat/tools/templates/leaflet.html
@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
 <head>
-  <title>RoboSat LeafLet Client</title>
+  <title>RoboSat Leaflet WebUI</title>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <link rel="stylesheet" href="https://unpkg.com/leaflet@1.3.4/dist/leaflet.css"
@@ -14,9 +14,15 @@
 <body>
   <div id="mapid" style="width:100%; height:100vh;"></div>
   <script>
-    var m = L.map("mapid").setView({{center}}, {{zoom}});
-    L.tileLayer("{{base_url}}/{z}/{x}/{y}.{{ext}}").addTo(m);
-    L.geoJSON({{grid}}, {style: {"color": "#cc0099", "opacity": 0.1, "fillOpacity": 0}}).addTo(m);
+function load_json(url, callback) {
+  if (url == '' ) return; var x = new XMLHttpRequest(); x.overrideMimeType("application/json"); x.open('GET', url, true);
+  x.onreadystatechange = function(){ if (x.readyState == 4 && x.status == "200") callback(JSON.parse(x.responseText)) };
+  x.send(null);
+};
+var m = L.map("mapid").setView({{center}}, {{zoom}});
+L.tileLayer("{{base_url}}/{z}/{x}/{y}.{{ext}}").addTo(m);
+var grid_style = {style: {"color": "#cc0099", "opacity": 0.3, "fillOpacity": 0}}
+load_json("{{tiles}}", function(grid){ L.geoJSON(grid, grid_style).addTo(m); });
   </script>
 </body>
 </html>

From 6aba3a7b153bb15674eec2664320c4c265a26380 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 20:59:07 +0100
Subject: [PATCH 73/97] New compare template. Able to perform prev/next and
 selection on images. And convert selection in coverage

---
 robosat/tools/templates/compare.html | 52 ++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 robosat/tools/templates/compare.html

diff --git a/robosat/tools/templates/compare.html b/robosat/tools/templates/compare.html
new file mode 100644
index 00000000..bce0b65a
--- /dev/null
+++ b/robosat/tools/templates/compare.html
@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>RoboSat Compare WebUI</title>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+</head>
+<body>
+  <div id="main" style="height:100vh; position:relative; text-align:center">
+    <p>Right Arrow: next image to compare, if any.</p>
+    <p>Left Arrow: previous image to compare, if any.</p>
+    <p>SpaceBar: select, or unselect, the current image.</p>
+    <p>Esc: ask to copy selected images, as a text cover, to clipboard.</p>
+  </div>
+  <script>
+class XYZ 
+{ constructor(json) { this.fts = json["features"]; this.i = 0; this.sel = Array.apply(false, Array(this.fts.length)) }
+  get()     { var p = this.fts[this.i]["properties"]; return { x: p.x, y: p.y, z: p.z } }
+  next()    { if ((1 + this.i - this.fts.length) <= 0) this.i++ ; return this.get(this.i) }
+  prev()    { if ((this.i - 1) >= 0) this.i-- ; return this.get(this.i) }
+  ping()    { this.sel[this.i] = this.sel[this.i] ? false : this.path() }
+  is_sel()  { return this.sel[this.i] }
+  get_sel() { var sel = ""; for (var i in this.sel) { if (this.sel[i]) { sel += this.sel[i] + " \n" } } return sel }
+  path()    { var xyz = this.get(); return xyz.z + "/" + xyz.x + "/" + xyz.y }
+  cover()   { var xyz = this.get(); return xyz.x + "," + xyz.y + "," + xyz.z }
+}
+function load_json(url, callback)
+{ if (url == "") return; var x = new XMLHttpRequest(); x.overrideMimeType("application/json"); x.open("GET", url, true);
+  x.onreadystatechange = function(){ if (x.readyState == 4 && x.status == "200") callback(JSON.parse(x.responseText)) };
+  x.send(null);
+}
+function display_img(xyz)
+{ document.getElementById("main").innerHTML = "<img id='image' src='" + xyz.path() + ".{{ext}}' />";
+  var style  = "position:absolute; width:auto; height:auto; margin:auto; top:0; bottom:0; left:0; right:0; "
+             + "vertical-align:middle; max-width:100%; max-height:100%";
+  document.getElementById("image").style = style;
+  if (xyz.is_sel()) { document.getElementById("image").style.border = "2px solid #cc0099" }
+               else { document.getElementById("image").style.border = "2px solid #ffffff" }
+}
+load_json("{{tiles}}", function(json)
+  { xyz = new XYZ(json); document.addEventListener("keydown", (e) =>
+    {
+      if (e.key == "ArrowRight")     { xyz.next() ; display_img(xyz) }
+      if (e.key == "ArrowLeft")      { xyz.prev() ; display_img(xyz) }
+      if (e.key == " " /* Space */ ) { xyz.ping() ; display_img(xyz) }
+      if (e.key == "Escape" )        { window.prompt("Copy selected tiles to clipboard: Ctrl+C, Enter", xyz.get_sel()) }
+    })
+  }
+)
+  </script>
+</body>
+</html>

From a0fd55571b69d7c66aa8eac5da0aa2ed51a7d7cb Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sat, 24 Nov 2018 23:36:55 +0100
Subject: [PATCH 74/97] remove tile_to_bbox, and use mercantile.xy_bounds
 instead

---
 robosat/tiles.py          | 16 ----------------
 robosat/tools/download.py |  5 +++--
 2 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index c2a38cba..1c2b6b32 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -48,22 +48,6 @@ def lerp(a, b, c):
     return lon, lat
 
 
-def tile_to_bbox(tile):
-    """Convert a tile to bbox coordinates
-
-    Args:
-      tile: the mercantile tile
-
-    Returns:
-       Tile's bbox coordinates (expressed in EPSG:3857)
-    """
-
-    west, south, east, north = mercantile.bounds(tile)
-    x, y = transform(CRS.from_epsg(4326), CRS.from_epsg(3857), [west, east], [north, south])
-
-    return [min(x), min(y), max(x), max(y)]
-
-
 def fetch_image(session, url, timeout=10):
     """Fetches the image representation for a tile.
 
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 2a3a4f78..fc0ceaba 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -7,8 +7,9 @@
 import requests
 from PIL import Image
 from tqdm import tqdm
+from mercantile import xy_bounds
 
-from robosat.tiles import tiles_from_csv, fetch_image, tile_to_bbox
+from robosat.tiles import tiles_from_csv, fetch_image
 from robosat.utils import web_ui
 from robosat.log import Log
 
@@ -67,7 +68,7 @@ def worker(tile):
                     tile.y = (2 ** tile.z) - tile.y - 1
                     url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
                 elif args.type == "WMS":
-                    xmin, ymin, xmax, ymax = tile_to_bbox(tile)
+                    xmin, ymin, xmax, ymax = xy_bounds(tile)
                     url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
                 else:
                     return tile, None, False

From d3dbd89acd0f672598b346b9b58764cd4a1e369b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 25 Nov 2018 00:50:44 +0100
Subject: [PATCH 75/97] Use the right cover separator

---
 robosat/tools/templates/compare.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tools/templates/compare.html b/robosat/tools/templates/compare.html
index bce0b65a..5881186e 100644
--- a/robosat/tools/templates/compare.html
+++ b/robosat/tools/templates/compare.html
@@ -18,7 +18,7 @@
   get()     { var p = this.fts[this.i]["properties"]; return { x: p.x, y: p.y, z: p.z } }
   next()    { if ((1 + this.i - this.fts.length) <= 0) this.i++ ; return this.get(this.i) }
   prev()    { if ((this.i - 1) >= 0) this.i-- ; return this.get(this.i) }
-  ping()    { this.sel[this.i] = this.sel[this.i] ? false : this.path() }
+  ping()    { this.sel[this.i] = this.sel[this.i] ? false : this.cover() }
   is_sel()  { return this.sel[this.i] }
   get_sel() { var sel = ""; for (var i in this.sel) { if (this.sel[i]) { sel += this.sel[i] + " \n" } } return sel }
   path()    { var xyz = this.get(); return xyz.z + "/" + xyz.x + "/" + xyz.y }

From 57f3bc466024da9c98539d9dd17e8be1c57fd86d Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Sun, 25 Nov 2018 14:54:41 +0100
Subject: [PATCH 76/97] Handle 1-N geojson features files. Few cleanup.

---
 robosat/tools/rasterize.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 3d55375e..5a1eb6fa 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -28,8 +28,8 @@ def add_parser(subparser):
         "rasterize", help="rasterize features to label masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("features", type=str, help="path to GeoJSON features file")
-    parser.add_argument("tiles", type=str, help="path to .csv tiles file")
+    parser.add_argument("features", type=str, nargs="+", help="path to GeoJSON features file")
+    parser.add_argument("cover", type=str, help="path to csv tiles cover file")
     parser.add_argument("out", type=str, help="directory to write converted images")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
@@ -88,10 +88,7 @@ def main(args):
     os.makedirs(args.out, exist_ok=True)
 
     # We can only rasterize all tiles at a single zoom.
-    assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
-
-    with open(args.features) as f:
-        fc = json.load(f)
+    assert all(tile.z == args.zoom for tile in tiles_from_csv(args.cover))
 
     # Find all tiles the features cover and make a map object for quick lookup.
     feature_map = collections.defaultdict(list)
@@ -124,29 +121,31 @@ def parse_geometry(feature_map, geometry, i):
 
         return feature_map
 
-    for i, feature in enumerate(tqdm(fc["features"], ascii=True, unit="feature")):
 
-        if feature["geometry"]["type"] == "GeometryCollection":
-            for geometry in feature["geometry"]["geometries"]:
-                feature_map = parse_geometry(feature_map, geometry, i)
-        else:
-            feature_map = parse_geometry(feature_map, feature["geometry"], i)
+    for feature in args.features:
+        with open(feature) as f:
+            fc = json.load(f)
+            for i, feature in enumerate(tqdm(fc["features"], ascii=True, unit="feature")):
+
+                if feature["geometry"]["type"] == "GeometryCollection":
+                    for geometry in feature["geometry"]["geometries"]:
+                        feature_map = parse_geometry(feature_map, geometry, i)
+                else:
+                    feature_map = parse_geometry(feature_map, feature["geometry"], i)
 
     # Burn features to tiles and write to a slippy map directory.
-    for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit="tile"):
+    for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True, unit="tile"):
         if tile in feature_map:
             out = burn(tile, feature_map[tile], args.size)
         else:
             out = Image.fromarray(np.zeros(shape=(args.size, args.size)).astype(int), mode="P")
 
-        palette = complementary_palette(make_palette(colors[0], colors[1]))
-        out.putpalette(palette)
-
         out_path = os.path.join(args.out, str(tile.z), str(tile.x))
         os.makedirs(out_path, exist_ok=True)
 
+        out.putpalette(complementary_palette(make_palette(colors[0], colors[1])))
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
     if args.web_ui:
-        tiles = [tile for tile in tiles_from_csv(args.tiles)]
+        tiles = [tile for tile in tiles_from_csv(args.cover)]
         web_ui(args.out, args.web_ui, tiles, tiles, "png", "leaflet.html")

From 01ea0b232ac0dfffe80de18904de8b63941d5f2b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 12:50:07 +0100
Subject: [PATCH 77/97] use args.parse choices

---
 robosat/tools/compare.py  | 5 +----
 robosat/tools/download.py | 4 +---
 robosat/tools/export.py   | 2 +-
 robosat/tools/tile.py     | 5 +----
 4 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 8af7468e..c7fa95a8 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -23,7 +23,7 @@ def add_parser(subparser):
     parser = subparser.add_parser(
         "compare", help="compare images and/or labels and masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    parser.add_argument("--mode", type=str, default="side", help="compare mode (e.g side, stack or list)")
+    parser.add_argument("--mode", type=str, default="side", choices=["side", "stack", "list"], help="compare mode")
     parser.add_argument("--images", type=str, nargs="+", help="slippy map images dirs to render (stack or side mode)")
     parser.add_argument("--ext", type=str, default="webp", help="file format to save images in (stack or side mode)")
     parser.add_argument("--labels", type=str, help="directory to read slippy map labels from (needed for QoD metric)")
@@ -150,9 +150,6 @@ def main(args):
             else:
                 out.write("{},{},{}\t\t{:.1f}\t\t{:.1f}{}".format(x, y, z, fg_ratio, qod, os.linesep))
 
-        else:
-            sys.exit("Unkown mode, should be either: side, stack or list")
-
     if args.mode == "list":
         if args.geojson:
             out.write("]}")
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index fc0ceaba..0e4c4e93 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -24,7 +24,7 @@ def add_parser(subparser):
     )
     parser.add_argument("--ext", type=str, default="webp", help="file format to save images in")
     parser.add_argument("--rate", type=int, default=10, help="rate limit in max. requests per second")
-    parser.add_argument("--type", type=str, default="XYZ", help="service type to use (e.g: XYZ, WMS or TMS)")
+    parser.add_argument("--type", type=str, default="XYZ", choices=["XYZ", "WMS", "TMS"], help="service type to use")
     parser.add_argument("--timeout", type=int, default=10, help="server request timeout (in seconds)")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
     parser.add_argument("tiles", type=str, help="path to .csv tiles file")
@@ -70,8 +70,6 @@ def worker(tile):
                 elif args.type == "WMS":
                     xmin, ymin, xmax, ymax = xy_bounds(tile)
                     url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
-                else:
-                    return tile, None, False
 
                 res = fetch_image(session, url, args.timeout)
                 if not res:
diff --git a/robosat/tools/export.py b/robosat/tools/export.py
index 8761c17e..5b1429a6 100644
--- a/robosat/tools/export.py
+++ b/robosat/tools/export.py
@@ -17,7 +17,7 @@ def add_parser(subparser):
 
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--export_channels", type=int, help="export channels to use (keep the first ones)")
-    parser.add_argument("--type", type=str, default="onnx", help="output type, i.e onnx or pth")
+    parser.add_argument("--type", type=str, choices=["onnx", "pth"], default="onnx", help="output type")
     parser.add_argument("--image_size", type=int, default=512, help="image size to use for model")
     parser.add_argument("--checkpoint", type=str, required=True, help="model checkpoint to load")
     parser.add_argument("out", type=str, help="path to save export model to")
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 1b3df199..4c068d52 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -29,7 +29,7 @@ def add_parser(subparser):
     parser.add_argument("out", type=str, help="directory to write tiles")
     parser.add_argument("--size", type=int, default=512, help="size of tiles side in pixels")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
-    parser.add_argument("--type", type=str, default="image", help="image or label tiling")
+    parser.add_argument("--type", type=str, choices=["image", "label"], default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
     parser.add_argument("--no_data", type=int, help="color considered as no data [0-255]. Skip related tile")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
@@ -115,9 +115,6 @@ def main(args):
                 ext = "webp"
                 Image.fromarray(np.moveaxis(data, 0, 2), mode="RGB").save("{}.{}".format(path, ext), optimize=True)
 
-        else:
-            sys.exit("Error: Unknown type, should be either 'image' or 'label'")
-
     if args.web_ui:
         tiles = [tile for tile in tiles if tile not in tiles_nodata]
         web_ui(args.out, args.web_ui, tiles, tiles, ext, "leaflet.html")

From b4391d3ad2ae49bb3d9be409660d822e5d85f182 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 12:50:50 +0100
Subject: [PATCH 78/97] Update to latest rasterio version

---
 deps/requirements-lock.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index 4ac56886..f75f7c8b 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -28,7 +28,7 @@ pyproj==1.9.5.1
 pytest==3.6.1
 python-dateutil==2.7.3
 pytz==2018.4
-rasterio==1.0.9
+rasterio==1.0.10
 requests==2.18.4
 Rtree==0.8.3
 scipy==1.1.0

From 3612d70c962158f4b5c9c3512506c61ff308c686 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 15:43:16 +0100
Subject: [PATCH 79/97] Dynamic handler arch.

---
 robosat/tools/extract.py  | 33 +++++++++++++++++-----------
 robosat/tools/features.py | 46 ++++++++++++++++++++-------------------
 2 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/robosat/tools/extract.py b/robosat/tools/extract.py
index 036a58cc..b4edb6fd 100644
--- a/robosat/tools/extract.py
+++ b/robosat/tools/extract.py
@@ -1,12 +1,9 @@
+import os
+import sys
 import argparse
-
-from robosat.osm.parking import ParkingHandler
-from robosat.osm.building import BuildingHandler
-from robosat.osm.road import RoadHandler
-
-# Register your osmium handlers here; in addition to the osmium handler interface
-# they need to support a `save(path)` function for GeoJSON serialization to a file.
-handlers = {"parking": ParkingHandler, "building": BuildingHandler, "road": RoadHandler}
+import pkgutil
+from pathlib import Path
+from importlib import import_module
 
 
 def add_parser(subparser):
@@ -16,14 +13,24 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("--type", type=str, required=True, choices=handlers.keys(), help="type of feature to extract")
-    parser.add_argument("map", type=str, help="path to .osm.pbf base map")
+    parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
+    parser.add_argument("pbf", type=str, help="path to .osm.pbf base map")
     parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")
 
     parser.set_defaults(func=main)
 
 
 def main(args):
-    handler = handlers[args.type]()
-    handler.apply_file(filename=args.map, locations=True)
-    handler.save(args.out)
+    module_path_search = [os.path.join(Path(__file__).parent.parent, "osm")]
+    modules = [name for _, name, _ in pkgutil.iter_modules(module_path_search) if name != "core"]
+    if args.type not in modules:
+        sys.exit("Unknown type, thoses available are {}".format(modules))
+
+    try:
+        module = import_module("robosat.osm." + args.type, package=__name__)
+        handler = getattr(module, "{}Handler".format(args.type.title()))
+        handler().apply_file(filename=args.pbf, locations=True)
+    except:
+        sys.exit("Something get wrong, unable to call {}Handler", args.type.title())
+
+    handler().save(args.out)
diff --git a/robosat/tools/features.py b/robosat/tools/features.py
index 062441ac..5e3f0e30 100644
--- a/robosat/tools/features.py
+++ b/robosat/tools/features.py
@@ -1,19 +1,14 @@
+import os
+import sys
 import argparse
-
+from tqdm import tqdm
 import numpy as np
-
 from PIL import Image
-from tqdm import tqdm
-
-from robosat.tiles import tiles_from_slippy_map
+import pkgutil
+from pathlib import Path
+from importlib import import_module
 from robosat.config import load_config
-
-from robosat.features.parking import ParkingHandler
-
-
-# Register post-processing handlers here; they need to support a `apply(tile, mask)` function
-# for handling one mask and a `save(path)` function for GeoJSON serialization to a file.
-handlers = {"parking": ParkingHandler}
+from robosat.tiles import tiles_from_slippy_map
 
 
 def add_parser(subparser):
@@ -23,9 +18,9 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("masks", type=str, help="slippy map directory with segmentation masks")
-    parser.add_argument("--type", type=str, required=True, choices=handlers.keys(), help="type of feature to extract")
+    parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("masks", type=str, help="slippy map directory with segmentation masks")
     parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")
 
     parser.set_defaults(func=main)
@@ -34,18 +29,25 @@ def add_parser(subparser):
 def main(args):
     dataset = load_config(args.dataset)
 
+    module_path_search = [os.path.join(Path(__file__).parent.parent, "features")]
+    modules = [name for _, name, _ in pkgutil.iter_modules(module_path_search) if name != "core"]
+    if args.type not in modules:
+        sys.exit("Unknown type, thoses available are {}".format(modules))
+
     labels = dataset["common"]["classes"]
-    assert set(labels).issuperset(set(handlers.keys())), "handlers have a class label"
+    if args.type not in labels:
+        sys.exit("The type you asked is not consistent with yours classes in the dataset file provided.")
     index = labels.index(args.type)
 
-    handler = handlers[args.type]()
-
-    tiles = list(tiles_from_slippy_map(args.masks))
+    try:
+        module = import_module("robosat.features." + args.type, package=__name__)
+        handler = getattr(module, "{}Handler".format(args.type.title()))
+    except:
+        sys.exit("Something get wrong, unable to call {}Handler", args.type.title())
 
-    for tile, path in tqdm(tiles, ascii=True, unit="mask"):
+    for tile, path in tqdm(list(tiles_from_slippy_map(args.masks)), ascii=True, unit="mask"):
         image = np.array(Image.open(path).convert("P"), dtype=np.uint8)
         mask = (image == index).astype(np.uint8)
+        handler().apply(tile, mask)
 
-        handler.apply(tile, mask)
-
-    handler.save(args.out)
+    handler().save(args.out)

From 0433e26722cf138d2f2fde04297406e35af32462 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 16:31:25 +0100
Subject: [PATCH 80/97] Use a single type parameter, more user understandable
 prototype

---
 robosat/tools/cover.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/robosat/tools/cover.py b/robosat/tools/cover.py
index 1a20534d..a07dd3af 100644
--- a/robosat/tools/cover.py
+++ b/robosat/tools/cover.py
@@ -18,9 +18,9 @@ def add_parser(subparser):
     )
 
     parser.add_argument("--zoom", type=int, help="zoom level of tiles")
-    parser.add_argument("--features", type=str, help="path to GeoJSON features")
-    parser.add_argument("--bbox", type=str, help="bbox expressed in lat/lon (i.e EPSG:4326)")
-    parser.add_argument("--path", type=str, help="Slippy Map directory input path")
+    parser.add_argument("--type", type=str, default="geojson", choices=["geojson", "bbox", "dir"], help="input type")
+    help = "input value, upon type either: a geojson file path, a bbox in lat/lon ESPG:4326, or a slippymap dir path"
+    parser.add_argument("input", type=str, help=help)
     parser.add_argument("out", type=str, help="path to csv file to store tiles in")
 
     parser.set_defaults(func=main)
@@ -28,27 +28,26 @@ def add_parser(subparser):
 
 def main(args):
 
-    if not args.zoom and (args.features or args.bbox):
+    if not args.zoom and args.type in ["geojson", "bbox"]:
         sys.exit("Zoom parameter is mandatory")
 
     cover = []
 
-    if args.features:
-        with open(args.features) as f:
+    if args.type == "geojson":
+        with open(args.input) as f:
             features = json.load(f)
 
         for feature in tqdm(features["features"], ascii=True, unit="feature"):
             cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist()))
 
-        # tiles can overlap for multiple features; unique tile ids
-        cover = list(set(cover))
+        cover = list(set(cover))  # tiles can overlap for multiple features; unique tile ids
 
-    elif args.bbox:
-        west, south, east, north = map(float, args.bbox.split(","))
+    elif args.type == "bbox":
+        west, south, east, north = map(float, args.input.split(","))
         cover = tiles(west, south, east, north, args.zoom)
 
-    elif args.path:
-        cover = [tile for tile, _ in tiles_from_slippy_map(args.path)]
+    elif args.type == "dir":
+        cover = [tile for tile, _ in tiles_from_slippy_map(args.input)]
 
     else:
         sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox, or an input directory path")

From ed16e8fa50820b32beb98083a58163bce8d61b2b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 17:52:27 +0100
Subject: [PATCH 81/97] add web_ui_template parameter. order args in tools to
 put optionals firsts

---
 robosat/tools/compare.py   |  7 +++++--
 robosat/tools/dedupe.py    |  4 ++--
 robosat/tools/download.py  |  4 +++-
 robosat/tools/masks.py     |  8 +++++---
 robosat/tools/merge.py     |  2 +-
 robosat/tools/predict.py   |  8 +++++---
 robosat/tools/rasterize.py | 10 ++++++----
 robosat/tools/subset.py    |  8 +++++---
 robosat/tools/tile.py      |  8 +++++---
 robosat/tools/train.py     |  2 +-
 robosat/utils.py           |  7 ++++++-
 11 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index c7fa95a8..32d59b55 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -36,6 +36,7 @@ def add_parser(subparser):
     parser.add_argument("--vertical", action="store_true", help="render vertical image aggregate, for side mode")
     parser.add_argument("--geojson", action="store_true", help="output geojson based, for list mode")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
     parser.add_argument("out", type=str, help="directory or path (upon mode) to save output to")
 
     parser.set_defaults(func=main)
@@ -156,8 +157,10 @@ def main(args):
         out.close()
 
     elif args.mode == "side" and args.web_ui:
-        web_ui(args.out, args.web_ui, None, tiles_compare, args.ext, "compare.html")
+        template = "compare.html" if not args.web_ui_template else args.web_ui_template
+        web_ui(args.out, args.web_ui, None, tiles_compare, args.ext, template)
 
     elif args.mode == "stack" and args.web_ui:
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile, _ in tiles_from_slippy_map(args.images[0])]
-        web_ui(args.out, args.web_ui, tiles, tiles_compare, args.ext, "leaflet.html")
+        web_ui(args.out, args.web_ui, tiles, tiles_compare, args.ext, template)
diff --git a/robosat/tools/dedupe.py b/robosat/tools/dedupe.py
index ab5d2445..fbd2c942 100644
--- a/robosat/tools/dedupe.py
+++ b/robosat/tools/dedupe.py
@@ -17,11 +17,11 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("osm", type=str, help="ground truth GeoJSON feature collection from OpenStreetMap")
-    parser.add_argument("predicted", type=str, help="predicted GeoJSON feature collection to deduplicate")
     parser.add_argument(
         "--threshold", type=float, required=True, help="maximum allowed IoU to keep predictions, between 0.0 and 1.0"
     )
+    parser.add_argument("osm", type=str, help="ground truth GeoJSON feature collection from OpenStreetMap")
+    parser.add_argument("predicted", type=str, help="predicted GeoJSON feature collection to deduplicate")
     parser.add_argument("out", type=str, help="path to GeoJSON to save deduplicated features to")
 
     parser.set_defaults(func=main)
diff --git a/robosat/tools/download.py b/robosat/tools/download.py
index 0e4c4e93..3d146dfe 100644
--- a/robosat/tools/download.py
+++ b/robosat/tools/download.py
@@ -27,6 +27,7 @@ def add_parser(subparser):
     parser.add_argument("--type", type=str, default="XYZ", choices=["XYZ", "WMS", "TMS"], help="service type to use")
     parser.add_argument("--timeout", type=int, default=10, help="server request timeout (in seconds)")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
     parser.add_argument("tiles", type=str, help="path to .csv tiles file")
     parser.add_argument("out", type=str, help="path to slippy map directory for storing tiles")
 
@@ -107,4 +108,5 @@ def worker(tile):
         log.log(" Coverage is fully downloaded.")
 
     if args.web_ui:
-        web_ui(args.out, args.web_ui, tiles, tiles, args.ext, "leaflet.html")
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
+        web_ui(args.out, args.web_ui, tiles, tiles, args.ext, template)
diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index f9f12ecc..2679248f 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -19,11 +19,12 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("masks", type=str, help="slippy map directory to save masks to")
-    parser.add_argument("probs", type=str, nargs="+", help="slippy map directories with class probabilities")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--weights", type=float, nargs="+", help="weights for weighted average soft-voting")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
+    parser.add_argument("masks", type=str, help="slippy map directory to save masks to")
+    parser.add_argument("probs", type=str, nargs="+", help="slippy map directories with class probabilities")
 
     parser.set_defaults(func=main)
 
@@ -74,8 +75,9 @@ def load(path):
         out.save(path, optimize=True)
 
     if args.web_ui:
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile, _ in list(tiles_from_slippy_map(args.probs[0]))]
-        web_ui(args.masks, args.web_ui, tiles, tiles, "png", "leaflet.html")
+        web_ui(args.masks, args.web_ui, tiles, tiles, "png", template)
 
 
 def softvote(probs, axis=0, weights=None):
diff --git a/robosat/tools/merge.py b/robosat/tools/merge.py
index e1e79f17..eac4e7ca 100644
--- a/robosat/tools/merge.py
+++ b/robosat/tools/merge.py
@@ -15,8 +15,8 @@ def add_parser(subparser):
         "merge", help="merged adjacent GeoJSON features", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("features", type=str, help="GeoJSON file to read features from")
     parser.add_argument("--threshold", type=int, required=True, help="minimum distance to adjacent features, in m")
+    parser.add_argument("features", type=str, help="GeoJSON file to read features from")
     parser.add_argument("out", type=str, help="path to GeoJSON to save merged features to")
 
     parser.set_defaults(func=main)
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 4ab5e078..8f3597cf 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -34,11 +34,12 @@ def add_parser(subparser):
     parser.add_argument("--overlap", type=int, default=32, help="tile pixel overlap to predict on")
     parser.add_argument("--tile_size", type=int, required=True, help="tile size for slippy map tiles")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
-    parser.add_argument("tiles", type=str, help="directory to read slippy map image tiles from")
-    parser.add_argument("probs", type=str, help="directory to save slippy map probability masks to")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
+    parser.add_argument("tiles", type=str, help="directory to read slippy map image tiles from")
+    parser.add_argument("probs", type=str, help="directory to save slippy map probability masks to")
 
     parser.set_defaults(func=main)
 
@@ -109,5 +110,6 @@ def map_location(storage, _):
                 out.save(path, optimize=True)
 
     if args.web_ui:
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile, _ in tiles_from_slippy_map(args.tiles)]
-        web_ui(args.probs, args.web_ui, tiles, tiles, "png", "leaflet.html")
+        web_ui(args.probs, args.web_ui, tiles, tiles, "png", template)
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 5a1eb6fa..01dfdc18 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -28,13 +28,14 @@ def add_parser(subparser):
         "rasterize", help="rasterize features to label masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("features", type=str, nargs="+", help="path to GeoJSON features file")
-    parser.add_argument("cover", type=str, help="path to csv tiles cover file")
-    parser.add_argument("out", type=str, help="directory to write converted images")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--size", type=int, default=512, help="size of rasterized image tiles in pixels")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
+    parser.add_argument("features", type=str, nargs="+", help="path to GeoJSON features file")
+    parser.add_argument("cover", type=str, help="path to csv tiles cover file")
+    parser.add_argument("out", type=str, help="directory to write converted images")
 
     parser.set_defaults(func=main)
 
@@ -147,5 +148,6 @@ def parse_geometry(feature_map, geometry, i):
         out.save(os.path.join(out_path, "{}.png".format(tile.y)), optimize=True)
 
     if args.web_ui:
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile in tiles_from_csv(args.cover)]
-        web_ui(args.out, args.web_ui, tiles, tiles, "png", "leaflet.html")
+        web_ui(args.out, args.web_ui, tiles, tiles, "png", template)
diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index 1c385b3a..9b152f4b 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -17,11 +17,12 @@ def add_parser(subparser):
         help="filter images in a slippy map directory using a csv",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
-    parser.add_argument("--web_ui", type=str, help="web ui base url")
-    parser.add_argument("--move", action="store_true", help="move files from src to dst (rather than copy them)" )
     parser.add_argument("dir", type=str, help="directory to read slippy map tiles from for filtering")
     parser.add_argument("cover", type=str, help="csv cover to filter tiles by")
     parser.add_argument("out", type=str, help="directory to save filtered tiles to")
+    parser.add_argument("--move", action="store_true", help="move files from src to dst (rather than copy them)" )
+    parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
 
     parser.set_defaults(func=main)
 
@@ -53,4 +54,5 @@ def main(args):
             sys.exit("Error: Unable to process {}".format(tile))
 
     if args.web_ui:
-        web_ui(args.out, args.web_ui, tiles, tiles, extension, "leaflet.html")
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
+        web_ui(args.out, args.web_ui, tiles, tiles, extension, template)
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 4c068d52..29fc852f 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -25,14 +25,15 @@ def add_parser(subparser):
         "tile", help="tile a raster image or label", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("raster", type=str, help="path to the raster to tile")
-    parser.add_argument("out", type=str, help="directory to write tiles")
     parser.add_argument("--size", type=int, default=512, help="size of tiles side in pixels")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, choices=["image", "label"], default="image", help="image or label tiling")
     parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
     parser.add_argument("--no_data", type=int, help="color considered as no data [0-255]. Skip related tile")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
+    parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
+    parser.add_argument("raster", type=str, help="path to the raster to tile")
+    parser.add_argument("out", type=str, help="directory to write tiles")
 
     parser.set_defaults(func=main)
 
@@ -116,5 +117,6 @@ def main(args):
                 Image.fromarray(np.moveaxis(data, 0, 2), mode="RGB").save("{}.{}".format(path, ext), optimize=True)
 
     if args.web_ui:
+        template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile in tiles if tile not in tiles_nodata]
-        web_ui(args.out, args.web_ui, tiles, tiles, ext, "leaflet.html")
+        web_ui(args.out, args.web_ui, tiles, tiles, ext, template)
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index a5ec4ae4..b433c468 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -43,12 +43,12 @@ def add_parser(subparser):
         "train", help="trains model on dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("out", type=str, help="directory to save checkpoint .pth files and log")
     parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
     parser.add_argument("--checkpoint", type=str, required=False, help="path to a model checkpoint (to retrain)")
     parser.add_argument("--resume", action="store_true", help="resume training (imply to provide a checkpoint)")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
+    parser.add_argument("out", type=str, help="directory to save checkpoint .pth files and log")
 
     parser.set_defaults(func=main)
 
diff --git a/robosat/utils.py b/robosat/utils.py
index 52ae60f6..c1a8dec6 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -1,7 +1,9 @@
 import re
 import os
+import sys
 import json
 import matplotlib
+from pathlib import Path
 from mercantile import feature
 from robosat.tiles import pixel_to_location
 
@@ -30,7 +32,10 @@ def plot(out, history):
 def web_ui(out, base_url, coverage_tiles, selected_tiles, ext, template):
 
     try:
-        web_ui = open(os.path.join("./robosat/tools/templates/" + template), "r").read()
+        if os.path.isfile(template):
+            web_ui = open(template, "r").read()
+        else:
+            web_ui = open(os.path.join(Path(__file__).parent, "tools", "templates", template), "r").read()
     except:
         sys.exit("Unable to open Web UI template {}".format(template))
 

From 53b0b4e5317453f50446cdd53ba7ce1fc37ad5c6 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 18:03:02 +0100
Subject: [PATCH 82/97] Protect already existing data in dest dir to be
 overwritten

---
 robosat/tools/subset.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index 9b152f4b..54d5b10f 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -44,7 +44,8 @@ def main(args):
         try:
             extension = os.path.splitext(src)[1][1:]
             dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension))
-            os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
+            if not os.path.isdir(os.path.join(args.out, str(tile.z), str(tile.x))):
+                os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
             if args.move:
                 assert(os.path.isfile(src))
                 shutil.move(src, dst)

From e02dffd63db124488e0ad8d73b28c2c571f988c1 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Mon, 26 Nov 2018 18:53:11 +0100
Subject: [PATCH 83/97] update existing tests to pass

---
 tests/test_datasets.py | 12 +++++++-----
 tests/test_tiles.py    |  5 ++++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/test_datasets.py b/tests/test_datasets.py
index 5c19b99f..50b4f629 100644
--- a/tests/test_datasets.py
+++ b/tests/test_datasets.py
@@ -30,22 +30,24 @@ def test_getitem_with_transform(self):
 
 class TestSlippyMapTilesConcatenation(unittest.TestCase):
     def test_len(self):
-        inputs = ["tests/fixtures/images/"]
+        path   = "tests/fixtures"
         target = "tests/fixtures/labels/"
+        channels = [{"sub": "images", "bands": [1, 2, 3]}]
 
         transform = JointCompose([JointTransform(ImageToTensor(), MaskToTensor())])
-        dataset = SlippyMapTilesConcatenation(inputs, target, transform)
+        dataset = SlippyMapTilesConcatenation(path, channels, target, transform)
 
         self.assertEqual(len(dataset), 3)
 
     def test_getitem(self):
-        inputs = ["tests/fixtures/images/"]
+        path   = "tests/fixtures"
         target = "tests/fixtures/labels/"
+        channels = [{"sub": "images", "bands": [1, 2, 3]}]
 
         transform = JointCompose([JointTransform(ImageToTensor(), MaskToTensor())])
-        dataset = SlippyMapTilesConcatenation(inputs, target, transform)
+        dataset = SlippyMapTilesConcatenation(path, channels, target, transform)
 
         images, mask, tiles = dataset[0]
-        self.assertEqual(tiles[0], mercantile.Tile(69105, 105093, 18))
+        self.assertEqual(tiles, mercantile.Tile(69105, 105093, 18))
         self.assertEqual(type(images), torch.Tensor)
         self.assertEqual(type(mask), torch.Tensor)
diff --git a/tests/test_tiles.py b/tests/test_tiles.py
index 68ba5443..8768a50d 100644
--- a/tests/test_tiles.py
+++ b/tests/test_tiles.py
@@ -9,6 +9,8 @@ class TestSlippyMapTiles(unittest.TestCase):
     def test_slippy_map_directory(self):
         root = "tests/fixtures/images"
         tiles = [tile for tile in tiles_from_slippy_map(root)]
+        tiles.sort()
+
         self.assertEqual(len(tiles), 3)
 
         tile, path = tiles[0]
@@ -20,6 +22,7 @@ class TestReadTiles(unittest.TestCase):
     def test_read_tiles(self):
         filename = "tests/fixtures/tiles.csv"
         tiles = [tile for tile in tiles_from_csv(filename)]
+        tiles.sort()
 
         self.assertEqual(len(tiles), 3)
-        self.assertEqual(tiles[0], mercantile.Tile(69623, 104945, 18))
+        self.assertEqual(tiles[1], mercantile.Tile(69623, 104945, 18))

From 1a451d3a2d05ac4908c56b4557e071a3331b58e2 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 27 Nov 2018 01:29:36 +0100
Subject: [PATCH 84/97] few cleanup. Add path parameter to allow an user to set
 his own extension dir

---
 robosat/tools/extract.py  | 30 +++++++++++++++++-------------
 robosat/tools/features.py | 31 +++++++++++++++++++------------
 2 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/robosat/tools/extract.py b/robosat/tools/extract.py
index b4edb6fd..356ae471 100644
--- a/robosat/tools/extract.py
+++ b/robosat/tools/extract.py
@@ -1,6 +1,7 @@
 import os
 import sys
 import argparse
+
 import pkgutil
 from pathlib import Path
 from importlib import import_module
@@ -14,6 +15,7 @@ def add_parser(subparser):
     )
 
     parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
+    parser.add_argument("--path", type=str, help="path to user's extension modules dir")
     parser.add_argument("pbf", type=str, help="path to .osm.pbf base map")
     parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")
 
@@ -21,16 +23,18 @@ def add_parser(subparser):
 
 
 def main(args):
-    module_path_search = [os.path.join(Path(__file__).parent.parent, "osm")]
-    modules = [name for _, name, _ in pkgutil.iter_modules(module_path_search) if name != "core"]
-    if args.type not in modules:
-        sys.exit("Unknown type, thoses available are {}".format(modules))
-
-    try:
-        module = import_module("robosat.osm." + args.type, package=__name__)
-        handler = getattr(module, "{}Handler".format(args.type.title()))
-        handler().apply_file(filename=args.pbf, locations=True)
-    except:
-        sys.exit("Something get wrong, unable to call {}Handler", args.type.title())
-
-    handler().save(args.out)
+    module_search_path = [args.path] if args.path else []
+    module_search_path.append(os.path.join(Path(__file__).parent.parent, "osm"))
+    modules = [(path, name) for path, name, _ in pkgutil.iter_modules(module_search_path) if name != "core"]
+    if args.type not in [name for _, name in modules]:
+        sys.exit("Unknown type, thoses available are {}".format([name for _, name in modules]))
+
+    if args.path:
+        sys.path.append(args.path)
+        module = import_module(args.type)
+    else:
+        module = import_module("robosat.osm.{}".format(args.type))
+
+    handler = getattr(module, "{}Handler".format(args.type.title()))()
+    handler.apply_file(filename=args.pbf, locations=True)
+    handler.save(args.out)
diff --git a/robosat/tools/features.py b/robosat/tools/features.py
index 5e3f0e30..019ae178 100644
--- a/robosat/tools/features.py
+++ b/robosat/tools/features.py
@@ -2,11 +2,14 @@
 import sys
 import argparse
 from tqdm import tqdm
+
 import numpy as np
 from PIL import Image
+
 import pkgutil
 from pathlib import Path
 from importlib import import_module
+
 from robosat.config import load_config
 from robosat.tiles import tiles_from_slippy_map
 
@@ -20,6 +23,7 @@ def add_parser(subparser):
 
     parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
     parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--path", type=str, help="path to user's extension modules dir")
     parser.add_argument("masks", type=str, help="slippy map directory with segmentation masks")
     parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")
 
@@ -27,27 +31,30 @@ def add_parser(subparser):
 
 
 def main(args):
-    dataset = load_config(args.dataset)
 
-    module_path_search = [os.path.join(Path(__file__).parent.parent, "features")]
-    modules = [name for _, name, _ in pkgutil.iter_modules(module_path_search) if name != "core"]
-    if args.type not in modules:
-        sys.exit("Unknown type, thoses available are {}".format(modules))
+    module_search_path = [args.path] if args.path else []
+    module_search_path.append(os.path.join(Path(__file__).parent.parent, "features"))
+    modules = [(path, name) for path, name, _ in pkgutil.iter_modules(module_search_path) if name != "core"]
+    if args.type not in [name for _, name in modules]:
+        sys.exit("Unknown type, thoses available are {}".format([name for _, name in modules]))
 
+    dataset = load_config(args.dataset)
     labels = dataset["common"]["classes"]
     if args.type not in labels:
         sys.exit("The type you asked is not consistent with yours classes in the dataset file provided.")
     index = labels.index(args.type)
 
-    try:
-        module = import_module("robosat.features." + args.type, package=__name__)
-        handler = getattr(module, "{}Handler".format(args.type.title()))
-    except:
-        sys.exit("Something get wrong, unable to call {}Handler", args.type.title())
+    if args.path:
+        sys.path.append(args.path)
+        module = import_module(args.type)
+    else:
+        module = import_module("robosat.features.{}".format(args.type))
+
+    handler = getattr(module, "{}Handler".format(args.type.title()))()
 
     for tile, path in tqdm(list(tiles_from_slippy_map(args.masks)), ascii=True, unit="mask"):
         image = np.array(Image.open(path).convert("P"), dtype=np.uint8)
         mask = (image == index).astype(np.uint8)
-        handler().apply(tile, mask)
+        handler.apply(tile, mask)
 
-    handler().save(args.out)
+    handler.save(args.out)

From 0fa123e2fbf37dbe49da84c3818ce8512de8c93b Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 27 Nov 2018 02:44:59 +0100
Subject: [PATCH 85/97] Black is black

---
 robosat/tools/rasterize.py | 1 -
 robosat/tools/subset.py    | 4 ++--
 robosat/utils.py           | 1 -
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 01dfdc18..8c53372f 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -122,7 +122,6 @@ def parse_geometry(feature_map, geometry, i):
 
         return feature_map
 
-
     for feature in args.features:
         with open(feature) as f:
             fc = json.load(f)
diff --git a/robosat/tools/subset.py b/robosat/tools/subset.py
index 54d5b10f..d47858e7 100644
--- a/robosat/tools/subset.py
+++ b/robosat/tools/subset.py
@@ -20,7 +20,7 @@ def add_parser(subparser):
     parser.add_argument("dir", type=str, help="directory to read slippy map tiles from for filtering")
     parser.add_argument("cover", type=str, help="csv cover to filter tiles by")
     parser.add_argument("out", type=str, help="directory to save filtered tiles to")
-    parser.add_argument("--move", action="store_true", help="move files from src to dst (rather than copy them)" )
+    parser.add_argument("--move", action="store_true", help="move files from src to dst (rather than copy them)")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
     parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
 
@@ -47,7 +47,7 @@ def main(args):
             if not os.path.isdir(os.path.join(args.out, str(tile.z), str(tile.x))):
                 os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
             if args.move:
-                assert(os.path.isfile(src))
+                assert os.path.isfile(src)
                 shutil.move(src, dst)
             else:
                 shutil.copyfile(src, dst)
diff --git a/robosat/utils.py b/robosat/utils.py
index c1a8dec6..34c64763 100644
--- a/robosat/utils.py
+++ b/robosat/utils.py
@@ -63,4 +63,3 @@ def web_ui(out, base_url, coverage_tiles, selected_tiles, ext, template):
                 fp.write('{}{{"type":"Feature",{},{}}}'.format("," if not first else "", geom, prop))
                 first = False
             fp.write("]}")
-

From 24716dbdc999511ab23cf2798b012ea4fca9d9d2 Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Tue, 27 Nov 2018 10:07:20 +0100
Subject: [PATCH 86/97] One rasterio version is enough ^^

---
 deps/requirements-lock.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deps/requirements-lock.txt b/deps/requirements-lock.txt
index f2ac5684..f0aa911a 100644
--- a/deps/requirements-lock.txt
+++ b/deps/requirements-lock.txt
@@ -29,7 +29,6 @@ pyproj==1.9.5.1
 pytest==3.9.1
 python-dateutil==2.7.3
 rasterio==1.0.10
-rasterio==1.0.8
 requests==2.20.0
 Rtree==0.8.3
 scikit-learn==0.20.0
@@ -44,4 +43,4 @@ torchvision==0.2.1
 tqdm==4.27.0
 urllib3==1.24
 Werkzeug==0.14.1
-webcolors==1.8.1
\ No newline at end of file
+webcolors==1.8.1

From bcf13ced15cec669c36c30373a12329c30b56662 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 27 Nov 2018 15:56:49 +0100
Subject: [PATCH 87/97] Create destination repository if needed

---
 robosat/tools/cover.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/robosat/tools/cover.py b/robosat/tools/cover.py
index a07dd3af..e91c005d 100644
--- a/robosat/tools/cover.py
+++ b/robosat/tools/cover.py
@@ -2,6 +2,7 @@
 import csv
 import json
 import sys
+import os
 
 from supermercado import burntiles
 from mercantile import tiles
@@ -52,6 +53,9 @@ def main(args):
     else:
         sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox, or an input directory path")
 
+    if not os.path.isdir(os.path.dirname(args.out)):
+        os.makedirs(os.path.dirname(args.out), exist_ok=True)
+
     with open(args.out, "w") as fp:
         writer = csv.writer(fp)
         writer.writerows(cover)

From 552ffb58c9377fde4d0ad550f8c32e8a1c8609fb Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 27 Nov 2018 17:40:58 +0100
Subject: [PATCH 88/97] add expanduser path

---
 robosat/tiles.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/robosat/tiles.py b/robosat/tiles.py
index 1c2b6b32..a1895b88 100644
--- a/robosat/tiles.py
+++ b/robosat/tiles.py
@@ -87,6 +87,7 @@ def isdigit(v):
         except ValueError:
             return False
 
+    root = os.path.expanduser(root)
     for z in os.listdir(root):
         if not isdigit(z):
             continue
@@ -116,6 +117,7 @@ def tiles_from_csv(path):
       The mercantile tiles from the csv file.
     """
 
+    path = os.path.expanduser(path)
     with open(path) as fp:
         reader = csv.reader(fp)
 
@@ -130,6 +132,7 @@ def tile_image(root, x, y, z):
     """Retrieves H,W,C numpy array, from a tile store and X,Y,Z coordinates, or `None`"""
 
     try:
+        root = os.path.expanduser(root)
         path = glob(os.path.join(root, z, x, y) + "*")
         assert len(path) == 1
         img = np.array(Image.open(path[0]).convert("RGB"))

From 0504df50cad8814849966d4663562fac807b16ca Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Tue, 27 Nov 2018 18:03:31 +0100
Subject: [PATCH 89/97] additional stdout log by default

---
 robosat/log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/log.py b/robosat/log.py
index 9f2ed9ae..7616ed08 100644
--- a/robosat/log.py
+++ b/robosat/log.py
@@ -10,7 +10,7 @@ class Log:
     """Create a log instance on a log file
     """
 
-    def __init__(self, path, out=None):
+    def __init__(self, path, out=sys.stdout):
 
         self.fp = None
         self.out = out

From 9f3eb8f964b1e8007cb56b56745601730189d802 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 28 Nov 2018 12:44:41 +0100
Subject: [PATCH 90/97] Add user hint

---
 robosat/tools/templates/compare.html | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/robosat/tools/templates/compare.html b/robosat/tools/templates/compare.html
index 5881186e..839054bd 100644
--- a/robosat/tools/templates/compare.html
+++ b/robosat/tools/templates/compare.html
@@ -37,13 +37,18 @@
   if (xyz.is_sel()) { document.getElementById("image").style.border = "2px solid #cc0099" }
                else { document.getElementById("image").style.border = "2px solid #ffffff" }
 }
+
+var msg_sel = `
+Copy selected tiles to clipboard: Ctrl+C, Enter.
+HINT: If any issue with your browser, use instead, from JS console: console.log(xyz.get_sel())`
+
 load_json("{{tiles}}", function(json)
   { xyz = new XYZ(json); document.addEventListener("keydown", (e) =>
     {
       if (e.key == "ArrowRight")     { xyz.next() ; display_img(xyz) }
       if (e.key == "ArrowLeft")      { xyz.prev() ; display_img(xyz) }
       if (e.key == " " /* Space */ ) { xyz.ping() ; display_img(xyz) }
-      if (e.key == "Escape" )        { window.prompt("Copy selected tiles to clipboard: Ctrl+C, Enter", xyz.get_sel()) }
+      if (e.key == "Escape" )        { window.prompt(msg_sel, xyz.get_sel()) }
     })
   }
 )

From 0aa7f46ce56603ca54dbd9318d28b5c7d9b5a505 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 28 Nov 2018 16:12:31 +0100
Subject: [PATCH 91/97] Allow to override most common train option from the
 command line (lr and epochs)

---
 robosat/tools/train.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index b95dbdef..379e6dd7 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -48,6 +48,8 @@ def add_parser(subparser):
     parser.add_argument("--checkpoint", type=str, required=False, help="path to a model checkpoint (to retrain)")
     parser.add_argument("--resume", action="store_true", help="resume training (imply to provide a checkpoint)")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
+    parser.add_argument("--epochs", type=int, help="if set, override epochs value from config file")
+    parser.add_argument("--lr", type=float, help="if set, override learning rate value from config file")
     parser.add_argument("out", type=str, help="directory to save checkpoint .pth files and log")
 
     parser.set_defaults(func=main)
@@ -81,7 +83,8 @@ def main(args):
         except KeyError:
             sys.exit("Error: The loss function used, need dataset weights values")
 
-    optimizer = Adam(net.parameters(), lr=model["opt"]["lr"], weight_decay=model["opt"]["decay"])
+    lr = args.lr if args.lr else model["opt"]["lr"]
+    optimizer = Adam(net.parameters(), lr=lr, weight_decay=model["opt"]["decay"])
 
     resume = 0
     if args.checkpoint:
@@ -111,7 +114,7 @@ def map_location(storage, _):
 
     train_loader, val_loader = get_dataset_loaders(model, dataset, args.workers)
 
-    num_epochs = model["opt"]["epochs"]
+    num_epochs = args.epochs if args.epochs else model["opt"]["epochs"]
     if resume >= num_epochs:
         sys.exit("Error: Epoch {} set in {} already reached by the checkpoint provided".format(num_epochs, args.model))
 
@@ -129,7 +132,7 @@ def map_location(storage, _):
     log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
     log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
     log.log("Data Augmentation:\t {}".format(model["opt"]["data_augmentation"]))
-    log.log("Learning Rate:\t\t {}".format(model["opt"]["lr"]))
+    log.log("Learning Rate:\t\t {}".format(lr))
     log.log("Weight Decay:\t\t {}".format(model["opt"]["decay"]))
     log.log("Loss function:\t\t {}".format(model["opt"]["loss"]))
     log.log("ResNet pre-trained:\t {}".format(model["opt"]["pretrained"]))

From d7e85594d7bb8f21267e643a86c27b2de8a2cceb Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 28 Nov 2018 21:22:15 +0100
Subject: [PATCH 92/97] Refactor config file, to a single one. Allow to
 override dataset path in train tool

---
 config/config.toml          | 55 +++++++++++++++++++++++++
 config/dataset-parking.toml | 30 --------------
 config/model-unet.toml      | 34 ---------------
 robosat/tools/compare.py    | 12 +++---
 robosat/tools/export.py     | 11 +++--
 robosat/tools/features.py   |  8 ++--
 robosat/tools/masks.py      |  6 +--
 robosat/tools/predict.py    |  8 ++--
 robosat/tools/rasterize.py  | 10 ++---
 robosat/tools/serve.py      | 14 +++----
 robosat/tools/tile.py       |  8 ++--
 robosat/tools/train.py      | 82 ++++++++++++++++++-------------------
 robosat/tools/weights.py    | 10 ++---
 13 files changed, 140 insertions(+), 148 deletions(-)
 create mode 100644 config/config.toml
 delete mode 100644 config/dataset-parking.toml
 delete mode 100644 config/model-unet.toml

diff --git a/config/config.toml b/config/config.toml
new file mode 100644
index 00000000..8546b1ca
--- /dev/null
+++ b/config/config.toml
@@ -0,0 +1,55 @@
+# RoboSat Configuration
+# For syntax see: https://github.com/toml-lang/toml#table-of-contents
+
+[dataset]
+  # The slippy map dataset's base directory.
+  path = '/tmp/slippy-map-dir/'
+
+  # Dataset specific class weights computes on the training data.
+  # Needed by 'mIoU' and 'CrossEntropy' losses to deal with unbalanced classes.
+  # Note: use `./rs weights -h` to compute these for new datasets.
+  weights = [1.6248, 5.762827]
+
+
+[classes]
+  # Human representation for classes.
+  titles = ['background', 'parking']
+
+  # Color map for visualization and representing classes in masks.
+  # Note: available colors are either CSS3 colors names or #RRGGBB hexadecimal representation.
+  colors  = ['denim', 'orange']
+
+
+# Channels configuration let your indicate wich dataset sub-directory and bands to take as input
+# You could so, add several channels blocks to compose your input Tensor. Orders are meaningful.
+[[channels]]
+sub = "images"
+bands = [1,2,3]
+
+
+# Model specific attributes.
+[model]
+
+  # Batch size for training.
+  batch_size = 2
+
+  # Image side size in pixels.
+  image_size = 512
+
+  # Total number of epochs to train for.
+  epochs     = 10
+
+  # Learning rate for the optimizer.
+  lr         = 0.0001
+
+  # Weight decay l2 penalty for the optimizer
+  decay      = 0.0001
+
+  # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
+  loss = 'Lovasz'
+
+  # Data augmentation, Flip or Rotate probability
+  data_augmentation = 0.75
+
+  # Use ImageNet weights pretraining
+  pretrained = true
diff --git a/config/dataset-parking.toml b/config/dataset-parking.toml
deleted file mode 100644
index 9e7065eb..00000000
--- a/config/dataset-parking.toml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Configuration related to a specific dataset.
-# For syntax see: https://github.com/toml-lang/toml#table-of-contents
-
-
-# Dataset specific common attributes.
-[common]
-
-  # The slippy map dataset's base directory.
-  dataset = '/tmp/slippy-map-dir/'
-
-  # Human representation for classes.
-  classes = ['background', 'parking']
-
-  # Color map for visualization and representing classes in masks.
-  # Note: available colors are either CSS3 colors names or #RRGGBB hexadecimal representation.
-  colors  = ['denim', 'orange']
-
-
-# Dataset specific class weights computes on the training data.
-# Needed by 'mIoU' and 'CrossEntropy' losses to deal with unbalanced classes.
-# Note: use `./rs weights -h` to compute these for new datasets.
-[weights]
-  values = [1.6248, 5.762827]
-
-
-# Channels configuration let your indicate wich dataset sub-directory and bands to take as input
-# You could so, add several channels blocks to compose your input Tensor. Orders are meaningful.
-[[channels]]
-sub = "images"
-bands = [1,2,3]
diff --git a/config/model-unet.toml b/config/model-unet.toml
deleted file mode 100644
index 802f17f3..00000000
--- a/config/model-unet.toml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Configuration related to a specific model.
-# For syntax see: https://github.com/toml-lang/toml#table-of-contents
-
-
-# Model specific common attributes.
-[common]
-
-  # Batch size for training.
-  batch_size = 2
-
-  # Image side size in pixels.
-  image_size = 512
-
-
-# Model specific optimization parameters.
-[opt]
-
-  # Total number of epochs to train for.
-  epochs     = 10
-
-  # Learning rate for the optimizer.
-  lr         = 0.0001
-
-  # Weight decay l2 penalty for the optimizer
-  decay      = 0.0001
-
-  # Loss function name (e.g 'Lovasz', 'mIoU' or 'CrossEntropy')
-  loss = 'Lovasz'
-
-  # Data augmentation, Flip or Rotate probability
-  data_augmentation = 0.75
-
-  # Use ImageNet weights pretraining
-  pretrained = true
diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 32d59b55..1a9c3f3f 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -28,7 +28,7 @@ def add_parser(subparser):
     parser.add_argument("--ext", type=str, default="webp", help="file format to save images in (stack or side mode)")
     parser.add_argument("--labels", type=str, help="directory to read slippy map labels from (needed for QoD metric)")
     parser.add_argument("--masks", type=str, help="directory to read slippy map masks from (needed for QoD metric)")
-    parser.add_argument("--dataset", type=str, help="path to dataset configuration file (needed for QoD metric)")
+    parser.add_argument("--config", type=str, help="path to configuration file (needed for QoD metric)")
     parser.add_argument("--minimum_fg", type=float, default=0.0, help="skip tile if label foreground below, [0-100]")
     parser.add_argument("--maximum_fg", type=float, default=100.0, help="skip tile if label foreground above, [0-100]")
     parser.add_argument("--minimum_qod", type=float, default=0.0, help="skip tile if QoD metric below, [0-100]")
@@ -66,11 +66,11 @@ def compare(masks, labels, tile, classes):
 
 def main(args):
 
-    if not args.masks or not args.labels or not args.dataset:
+    if not args.masks or not args.labels or not args.config:
         if args.mode == "list":
-            sys.exit("Parameters masks, labels and dataset, are all mandatories in list mode.")
+            sys.exit("Parameters masks, labels and config, are all mandatories in list mode.")
         if args.minimum_fg > 0 or args.maximum_fg < 100 or args.minimum_qod > 0 or args.maximum_qod < 100:
-            sys.exit("Parameters masks, labels and dataset, are all mandatories in QoD filtering.")
+            sys.exit("Parameters masks, labels and config, are all mandatories in QoD filtering.")
 
     if args.images:
         tiles = [tile for tile, _ in tiles_from_slippy_map(args.images[0])]
@@ -97,8 +97,8 @@ def main(args):
 
         x, y, z = list(map(str, tile))
 
-        if args.masks and args.labels and args.dataset:
-            classes = load_config(args.dataset)["common"]["classes"]
+        if args.masks and args.labels and args.config:
+            classes = load_config(args.config)["classes"]["classes"]
             dist, fg_ratio, qod = compare(args.masks, args.labels, tile, classes)
             if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod:
                 continue
diff --git a/robosat/tools/export.py b/robosat/tools/export.py
index 5b1429a6..5887d827 100644
--- a/robosat/tools/export.py
+++ b/robosat/tools/export.py
@@ -15,7 +15,7 @@ def add_parser(subparser):
         "export", help="exports or prunes model", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--export_channels", type=int, help="export channels to use (keep the first ones)")
     parser.add_argument("--type", type=str, choices=["onnx", "pth"], default="onnx", help="output type")
     parser.add_argument("--image_size", type=int, default=512, help="image size to use for model")
@@ -26,14 +26,17 @@ def add_parser(subparser):
 
 
 def main(args):
-    dataset = load_config(args.dataset)
+    config = load_config(args.config)
 
     if args.type == "onnx":
         os.environ["CUDA_VISIBLE_DEVICES"] = ""
         # Workaround: PyTorch ONNX, DataParallel with GPU issue, cf https://github.com/pytorch/pytorch/issues/5315
 
-    num_classes = len(dataset["common"]["classes"])
-    num_channels = len(dataset["common"]["channels"])
+    num_classes = len(config["classes"]["classes"])
+    num_channels = 0
+    for channel in config["channels"]:
+        num_channels += len(channel["bands"])
+
     export_channels = num_channels if not args.export_channels else args.export_channels
     assert num_channels >= export_channels, "Will be hard indeed, to export more channels than thoses dataset provide"
 
diff --git a/robosat/tools/features.py b/robosat/tools/features.py
index 019ae178..796b59c6 100644
--- a/robosat/tools/features.py
+++ b/robosat/tools/features.py
@@ -22,7 +22,7 @@ def add_parser(subparser):
     )
 
     parser.add_argument("--type", type=str, required=True, help="type of feature to extract")
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--path", type=str, help="path to user's extension modules dir")
     parser.add_argument("masks", type=str, help="slippy map directory with segmentation masks")
     parser.add_argument("out", type=str, help="path to GeoJSON file to store features in")
@@ -38,10 +38,10 @@ def main(args):
     if args.type not in [name for _, name in modules]:
         sys.exit("Unknown type, thoses available are {}".format([name for _, name in modules]))
 
-    dataset = load_config(args.dataset)
-    labels = dataset["common"]["classes"]
+    config = load_config(args.config)
+    labels = config["classes"]["titles"]
     if args.type not in labels:
-        sys.exit("The type you asked is not consistent with yours classes in the dataset file provided.")
+        sys.exit("The type you asked is not consistent with yours classes in the config file provided.")
     index = labels.index(args.type)
 
     if args.path:
diff --git a/robosat/tools/masks.py b/robosat/tools/masks.py
index 2679248f..dd974c86 100644
--- a/robosat/tools/masks.py
+++ b/robosat/tools/masks.py
@@ -19,7 +19,7 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--weights", type=float, nargs="+", help="weights for weighted average soft-voting")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
     parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
@@ -63,8 +63,8 @@ def load(path):
         mask = softvote(probs, axis=0, weights=args.weights)
         mask = mask.astype(np.uint8)
 
-        dataset = load_config(args.dataset)
-        palette = make_palette(dataset["common"]["colors"][0], dataset["common"]["colors"][1])
+        config = load_config(args.config)
+        palette = make_palette(config["classes"]["colors"][0], config["classes"]["colors"][1])
 
         out = Image.fromarray(mask, mode="P")
         out.putpalette(palette)
diff --git a/robosat/tools/predict.py b/robosat/tools/predict.py
index 8f3597cf..64251b66 100644
--- a/robosat/tools/predict.py
+++ b/robosat/tools/predict.py
@@ -34,7 +34,7 @@ def add_parser(subparser):
     parser.add_argument("--overlap", type=int, default=32, help="tile pixel overlap to predict on")
     parser.add_argument("--tile_size", type=int, required=True, help="tile size for slippy map tiles")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--masks_output", action="store_true", help="output masks rather than probs")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
     parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
@@ -45,8 +45,8 @@ def add_parser(subparser):
 
 
 def main(args):
-    dataset = load_config(args.dataset)
-    num_classes = len(dataset["common"]["classes"])
+    config = load_config(args.config)
+    num_classes = len(config["classes"]["titles"])
 
     if torch.cuda.is_available():
         device = torch.device("cuda")
@@ -74,7 +74,7 @@ def map_location(storage, _):
     loader = DataLoader(directory, batch_size=args.batch_size, num_workers=args.workers)
 
     if args.masks_output:
-        palette = make_palette(dataset["common"]["colors"][0], dataset["common"]["colors"][1])
+        palette = make_palette(config["classes"]["colors"][0], config["classes"]["colors"][1])
     else:
         palette = continuous_palette_for_color("pink", 256)
 
diff --git a/robosat/tools/rasterize.py b/robosat/tools/rasterize.py
index 122ceed3..04749816 100644
--- a/robosat/tools/rasterize.py
+++ b/robosat/tools/rasterize.py
@@ -28,7 +28,7 @@ def add_parser(subparser):
         "rasterize", help="rasterize features to label masks", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--size", type=int, default=512, help="size of rasterized image tiles in pixels")
     parser.add_argument("--web_ui", type=str, help="web ui client base url")
@@ -78,10 +78,10 @@ def burn(tile, features, size, burn_value=1):
 
 
 def main(args):
-    dataset = load_config(args.dataset)
+    config = load_config(args.config)
 
-    classes = dataset["common"]["classes"]
-    colors = dataset["common"]["colors"]
+    classes = config["classes"]["titles"]
+    colors = config["classes"]["colors"]
     assert len(classes) == len(colors), "classes and colors coincide"
     assert len(colors) == 2, "only binary models supported right now"
 
@@ -159,4 +159,4 @@ def parse_geometry(feature_map, geometry, i):
     if args.web_ui:
         template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
         tiles = [tile for tile in tiles_from_csv(args.cover)]
-        web_ui(args.out, args.web_ui, tiles, tiles, "png", template)
\ No newline at end of file
+        web_ui(args.out, args.web_ui, tiles, tiles, "png", template)
diff --git a/robosat/tools/serve.py b/robosat/tools/serve.py
index ea23cab5..a815d280 100644
--- a/robosat/tools/serve.py
+++ b/robosat/tools/serve.py
@@ -84,7 +84,7 @@ def add_parser(subparser):
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
 
     parser.add_argument("--url", type=str, help="endpoint with {z}/{x}/{y} variables to fetch image tiles from")
     parser.add_argument("--checkpoint", type=str, required=True, help="model checkpoint to load")
@@ -96,7 +96,7 @@ def add_parser(subparser):
 
 
 def main(args):
-    dataset = load_config(args.dataset)
+    config = load_config(args.config)
 
     global size
     size = args.tile_size
@@ -114,7 +114,7 @@ def main(args):
     tiles = args.url
 
     global predictor
-    predictor = Predictor(args.checkpoint, dataset)
+    predictor = Predictor(args.checkpoint, config)
 
     app.run(host=args.host, port=args.port, threaded=False)
 
@@ -127,13 +127,13 @@ def send_png(image):
 
 
 class Predictor:
-    def __init__(self, checkpoint, dataset):
+    def __init__(self, checkpoint, config):
 
         self.cuda = torch.cuda.is_available()
         self.device = torch.device("cuda" if self.cuda else "cpu")
 
         self.checkpoint = checkpoint
-        self.dataset = dataset
+        self.config = config
 
         self.net = self.net_from_chkpt_()
 
@@ -156,7 +156,7 @@ def segment(self, image):
 
             mask = Image.fromarray(mask, mode="P")
 
-            palette = make_palette(*self.dataset["common"]["colors"])
+            palette = make_palette(*self.config["common"]["colors"])
             mask.putpalette(palette)
 
             return mask
@@ -168,7 +168,7 @@ def map_location(storage, _):
         # https://github.com/pytorch/pytorch/issues/7178
         chkpt = torch.load(self.checkpoint, map_location=map_location)
 
-        num_classes = len(self.dataset["common"]["classes"])
+        num_classes = len(self.config["classes"]["titles"])
 
         net = UNet(num_classes).to(self.device)
         net = nn.DataParallel(net)
diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index 29fc852f..c792491c 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -28,7 +28,7 @@ def add_parser(subparser):
     parser.add_argument("--size", type=int, default=512, help="size of tiles side in pixels")
     parser.add_argument("--zoom", type=int, required=True, help="zoom level of tiles")
     parser.add_argument("--type", type=str, choices=["image", "label"], default="image", help="image or label tiling")
-    parser.add_argument("--dataset", type=str, help="path to dataset configuration file, mandatory for label tiling")
+    parser.add_argument("--config", type=str, help="path to configuration file, mandatory for label tiling")
     parser.add_argument("--no_data", type=int, help="color considered as no data [0-255]. Skip related tile")
     parser.add_argument("--web_ui", type=str, help="web ui base url")
     parser.add_argument("--web_ui_template", type=str, help="path to an alternate web ui template")
@@ -42,12 +42,12 @@ def main(args):
 
     if args.type == "label":
         try:
-            dataset = load_config(args.dataset)
+            config = load_config(args.config)
         except:
             sys.exit("Error: Unable to load DataSet config file")
 
-        classes = dataset["common"]["classes"]
-        colors = dataset["common"]["colors"]
+        classes = config["classes"]["title"]
+        colors = config["classes"]["colors"]
         assert len(classes) == len(colors), "classes and colors coincide"
         assert len(colors) == 2, "only binary models supported right now"
 
diff --git a/robosat/tools/train.py b/robosat/tools/train.py
index 379e6dd7..954a7aeb 100644
--- a/robosat/tools/train.py
+++ b/robosat/tools/train.py
@@ -43,11 +43,11 @@ def add_parser(subparser):
         "train", help="trains model on dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("--model", type=str, required=True, help="path to model configuration file")
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
     parser.add_argument("--checkpoint", type=str, required=False, help="path to a model checkpoint (to retrain)")
     parser.add_argument("--resume", action="store_true", help="resume training (imply to provide a checkpoint)")
     parser.add_argument("--workers", type=int, default=0, help="number of workers pre-processing images")
+    parser.add_argument("--dataset", type=int, help="if set, override dataset path value from config file")
     parser.add_argument("--epochs", type=int, help="if set, override epochs value from config file")
     parser.add_argument("--lr", type=float, help="if set, override learning rate value from config file")
     parser.add_argument("out", type=str, help="directory to save checkpoint .pth files and log")
@@ -56,35 +56,36 @@ def add_parser(subparser):
 
 
 def main(args):
-    model = load_config(args.model)
-    dataset = load_config(args.dataset)
+    config = load_config(args.config)
+    lr = args.lr if args.lr else config["model"]["lr"]
+    dataset_path = args.dataset if args.dataset else config["dataset"]["path"]
+    num_epochs = args.epochs if args.epochs else config["model"]["epochs"]
 
     log = Log(os.path.join(args.out, "log"))
 
     if torch.cuda.is_available():
         device = torch.device("cuda")
-      
+
         torch.backends.cudnn.benchmark = True
         log.log("RoboSat - training on {} GPUs, with {} workers".format(torch.cuda.device_count(), args.workers))
     else:
         device = torch.device("cpu")
         log.log("RoboSat - training on CPU, with {} workers", format(args.workers))
 
-    num_classes = len(dataset["common"]["classes"])
+    num_classes = len(config["classes"]["titles"])
     num_channels = 0
-    for channel in dataset["channels"]:
+    for channel in config["channels"]:
         num_channels += len(channel["bands"])
-    pretrained = model["opt"]["pretrained"]
+    pretrained = config["model"]["pretrained"]
     net = DataParallel(UNet(num_classes, num_channels=num_channels, pretrained=pretrained)).to(device)
 
-    if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
+    if config["model"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
         try:
-            weight = torch.Tensor(dataset["weights"]["values"])
+            weight = torch.Tensor(config["classes"]["weights"])
         except KeyError:
             sys.exit("Error: The loss function used, need dataset weights values")
 
-    lr = args.lr if args.lr else model["opt"]["lr"]
-    optimizer = Adam(net.parameters(), lr=lr, weight_decay=model["opt"]["decay"])
+    optimizer = Adam(net.parameters(), lr=lr, weight_decay=config["model"]["decay"])
 
     resume = 0
     if args.checkpoint:
@@ -101,43 +102,42 @@ def map_location(storage, _):
             optimizer.load_state_dict(chkpt["optimizer"])
             resume = chkpt["epoch"]
 
-    if model["opt"]["loss"] == "CrossEntropy":
+    if config["model"]["loss"] == "CrossEntropy":
         criterion = CrossEntropyLoss2d(weight=weight).to(device)
-    elif model["opt"]["loss"] == "mIoU":
+    elif config["model"]["loss"] == "mIoU":
         criterion = mIoULoss2d(weight=weight).to(device)
-    elif model["opt"]["loss"] == "Focal":
+    elif config["model"]["loss"] == "Focal":
         criterion = FocalLoss2d(weight=weight).to(device)
-    elif model["opt"]["loss"] == "Lovasz":
+    elif config["model"]["loss"] == "Lovasz":
         criterion = LovaszLoss2d().to(device)
     else:
-        sys.exit("Error: Unknown [opt][loss] value !")
+        sys.exit("Error: Unknown [model][loss] value !")
 
-    train_loader, val_loader = get_dataset_loaders(model, dataset, args.workers)
+    train_loader, val_loader = get_dataset_loaders(dataset_path, config, args.workers)
 
-    num_epochs = args.epochs if args.epochs else model["opt"]["epochs"]
     if resume >= num_epochs:
-        sys.exit("Error: Epoch {} set in {} already reached by the checkpoint provided".format(num_epochs, args.model))
+        sys.exit("Error: Epoch {} set in {} already reached by the checkpoint provided".format(num_epochs, args.config))
 
     history = collections.defaultdict(list)
 
     log.log("")
-    log.log("--- Input tensor from Dataset: {} ---".format(dataset["common"]["dataset"]))
+    log.log("--- Input tensor from Dataset: {} ---".format(dataset_path))
     num_channel = 1
-    for channel in dataset["channels"]:
+    for channel in config["channels"]:
         for band in channel["bands"]:
             log.log("Channel {}:\t\t {}[band: {}]".format(num_channel, channel["sub"], band))
             num_channel += 1
     log.log("")
     log.log("--- Hyper Parameters ---")
-    log.log("Batch Size:\t\t {}".format(model["common"]["batch_size"]))
-    log.log("Image Size:\t\t {}".format(model["common"]["image_size"]))
-    log.log("Data Augmentation:\t {}".format(model["opt"]["data_augmentation"]))
+    log.log("Batch Size:\t\t {}".format(config["model"]["batch_size"]))
+    log.log("Image Size:\t\t {}".format(config["model"]["image_size"]))
+    log.log("Data Augmentation:\t {}".format(config["model"]["data_augmentation"]))
     log.log("Learning Rate:\t\t {}".format(lr))
-    log.log("Weight Decay:\t\t {}".format(model["opt"]["decay"]))
-    log.log("Loss function:\t\t {}".format(model["opt"]["loss"]))
-    log.log("ResNet pre-trained:\t {}".format(model["opt"]["pretrained"]))
+    log.log("Weight Decay:\t\t {}".format(config["model"]["decay"]))
+    log.log("Loss function:\t\t {}".format(config["model"]["loss"]))
+    log.log("ResNet pre-trained:\t {}".format(config["model"]["pretrained"]))
     if "weight" in locals():
-        log.log("Weights :\t\t {}".format(dataset["weights"]["values"]))
+        log.log("Weights :\t\t {}".format(config["dataset"]["weights"]))
     log.log("")
 
     for epoch in range(resume, num_epochs):
@@ -150,7 +150,7 @@ def map_location(storage, _):
             "Train    loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format(
                 train_hist["loss"],
                 train_hist["miou"],
-                dataset["common"]["classes"][1],
+                config["classes"]["titles"][1],
                 train_hist["fg_iou"],
                 train_hist["mcc"],
             )
@@ -162,7 +162,7 @@ def map_location(storage, _):
         val_hist = validate(val_loader, num_classes, device, net, criterion)
         log.log(
             "Validate loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format(
-                val_hist["loss"], val_hist["miou"], dataset["common"]["classes"][1], val_hist["fg_iou"], val_hist["mcc"]
+                val_hist["loss"], val_hist["miou"], config["classes"]["titles"][1], val_hist["fg_iou"], val_hist["mcc"]
             )
         )
 
@@ -258,35 +258,35 @@ def validate(loader, num_classes, device, net, criterion):
     }
 
 
-def get_dataset_loaders(model, dataset, workers):
+def get_dataset_loaders(path, config, workers):
 
     # Values computed on ImageNet DataSet
     mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
 
     transform = JointCompose(
         [
-            JointResize(model["common"]["image_size"]),
-            JointRandomFlipOrRotate(model["opt"]["data_augmentation"]),
+            JointResize(config["model"]["image_size"]),
+            JointRandomFlipOrRotate(config["model"]["data_augmentation"]),
             JointTransform(ImageToTensor(), MaskToTensor()),
             JointTransform(Normalize(mean=mean, std=std), None),
         ]
     )
 
     train_dataset = SlippyMapTilesConcatenation(
-        os.path.join(dataset["common"]["dataset"], "training"),
-        dataset["channels"],
-        os.path.join(dataset["common"]["dataset"], "training", "labels"),
+        os.path.join(path, "training"),
+        config["channels"],
+        os.path.join(path, "training", "labels"),
         joint_transform=transform,
     )
 
     val_dataset = SlippyMapTilesConcatenation(
-        os.path.join(dataset["common"]["dataset"], "validation"),
-        dataset["channels"],
-        os.path.join(dataset["common"]["dataset"], "validation", "labels"),
+        os.path.join(path, "validation"),
+        config["channels"],
+        os.path.join(path, "validation", "labels"),
         joint_transform=transform,
     )
 
-    batch_size = model["common"]["batch_size"]
+    batch_size = config["model"]["batch_size"]
     train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=workers)
     val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, drop_last=True, num_workers=workers)
 
diff --git a/robosat/tools/weights.py b/robosat/tools/weights.py
index 9caf11e2..af68dd1e 100644
--- a/robosat/tools/weights.py
+++ b/robosat/tools/weights.py
@@ -18,19 +18,17 @@ def add_parser(subparser):
         "weights", help="computes class weights on dataset", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
 
-    parser.add_argument("--dataset", type=str, required=True, help="path to dataset configuration file")
+    parser.add_argument("--config", type=str, required=True, help="path to configuration file")
 
     parser.set_defaults(func=main)
 
 
 def main(args):
-    dataset = load_config(args.dataset)
-
-    path = dataset["common"]["dataset"]
-    num_classes = len(dataset["common"]["classes"])
+    config = load_config(args.config)
+    path = config["dataset"]["path"]
+    num_classes = len(config["classes"]["titles"])
 
     train_transform = Compose([MaskToTensor()])
-
     train_dataset = SlippyMapTiles(os.path.join(path, "training", "labels"), "mask", transform=train_transform)
 
     n = 0

From c62897569ab11b795bb55f28bd05ac1383cf8d6d Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Wed, 28 Nov 2018 22:15:22 +0100
Subject: [PATCH 93/97] polish

---
 robosat/tools/cover.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/robosat/tools/cover.py b/robosat/tools/cover.py
index e91c005d..0c58ea42 100644
--- a/robosat/tools/cover.py
+++ b/robosat/tools/cover.py
@@ -1,12 +1,12 @@
+import os
+import sys
 import argparse
 import csv
 import json
-import sys
-import os
 
-from supermercado import burntiles
-from mercantile import tiles
 from tqdm import tqdm
+from mercantile import tiles
+from supermercado import burntiles
 
 from robosat.datasets import tiles_from_slippy_map
 
@@ -14,15 +14,15 @@
 def add_parser(subparser):
     parser = subparser.add_parser(
         "cover",
-        help="generates tiles covering GeoJSON features or lat/lon Bbox",
+        help="generates tiles covering, in csv format: X,Y,Z",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
 
     parser.add_argument("--zoom", type=int, help="zoom level of tiles")
     parser.add_argument("--type", type=str, default="geojson", choices=["geojson", "bbox", "dir"], help="input type")
-    help = "input value, upon type either: a geojson file path, a bbox in lat/lon ESPG:4326, or a slippymap dir path"
+    help = "input value, upon type either: a geojson file path, a lat/lon bbox in ESPG:4326, or a slippymap dir path"
     parser.add_argument("input", type=str, help=help)
-    parser.add_argument("out", type=str, help="path to csv file to store tiles in")
+    parser.add_argument("out", type=str, help="path to csv file to generate")
 
     parser.set_defaults(func=main)
 
@@ -50,12 +50,8 @@ def main(args):
     elif args.type == "dir":
         cover = [tile for tile, _ in tiles_from_slippy_map(args.input)]
 
-    else:
-        sys.exit("You have to provide either a GeoJson features file, or a lat/lon bbox, or an input directory path")
-
     if not os.path.isdir(os.path.dirname(args.out)):
         os.makedirs(os.path.dirname(args.out), exist_ok=True)
 
     with open(args.out, "w") as fp:
-        writer = csv.writer(fp)
-        writer.writerows(cover)
+        csv.writer(fp).writerows(cover)

From a59d971f8593943691ecc5a72132d4c12634a579 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Thu, 29 Nov 2018 01:01:37 +0100
Subject: [PATCH 94/97] Add geojson slicer

---
 robosat/tools/templates/leaflet.html | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/robosat/tools/templates/leaflet.html b/robosat/tools/templates/leaflet.html
index 4867ec0b..2fe49cac 100644
--- a/robosat/tools/templates/leaflet.html
+++ b/robosat/tools/templates/leaflet.html
@@ -4,12 +4,10 @@
   <title>RoboSat Leaflet WebUI</title>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.3.4/dist/leaflet.css"
-        integrity="sha512-puBpdR0798OZvTTbP4A8Ix/l+A4dHDD0DGqYW6RQ+9jxkRFclaxxQb/SJAWZfWAkuyeQUytO7+7N4QKrDh+drA=="
-	crossorigin="" />
-  <script src="https://unpkg.com/leaflet@1.3.4/dist/leaflet.js"
-	  integrity="sha512-nMMmRyTVoLYqjP9hrbed9S+FzjZHW5gY1TWCHA5ckwXZBadntCNs8kEqAWdrb9O7rxbCaA4lKTIWjDXZxflOcA=="
-	  crossorigin=""></script>
+  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.3.4/dist/leaflet.css" />
+  <script src="https://unpkg.com/leaflet@1.3.4/dist/leaflet.js"></script>
+  <script src="https://unpkg.com/geojson-vt@3.2.0/geojson-vt.js"></script>
+  <script src="https://unpkg.com/leaflet.vectorgrid@latest/dist/Leaflet.VectorGrid.js"></script>
 </head>
 <body>
   <div id="mapid" style="width:100%; height:100vh;"></div>
@@ -21,8 +19,12 @@
 };
 var m = L.map("mapid").setView({{center}}, {{zoom}});
 L.tileLayer("{{base_url}}/{z}/{x}/{y}.{{ext}}").addTo(m);
-var grid_style = {style: {"color": "#cc0099", "opacity": 0.3, "fillOpacity": 0}}
-load_json("{{tiles}}", function(grid){ L.geoJSON(grid, grid_style).addTo(m); });
+
+var grid_style = { "color": "deeppink", "opacity": 0.3, "fill": false }
+load_json("{{tiles}}", function(grid){
+    L.vectorGrid.slicer( grid, { maxZoom: {{zoom}},
+                                 rendererFactory: L.canvas.tile,
+                                 vectorTileLayerStyles: { sliced: grid_style } } ).addTo(m) })
   </script>
 </body>
 </html>

From 9b8f94ec4339ed8208dad4063a294e22b88c4683 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 30 Nov 2018 10:09:43 +0100
Subject: [PATCH 95/97] Fix: propagate config classes titles

---
 robosat/tools/compare.py | 2 +-
 robosat/tools/export.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/robosat/tools/compare.py b/robosat/tools/compare.py
index 1a9c3f3f..4b2f3c05 100644
--- a/robosat/tools/compare.py
+++ b/robosat/tools/compare.py
@@ -98,7 +98,7 @@ def main(args):
         x, y, z = list(map(str, tile))
 
         if args.masks and args.labels and args.config:
-            classes = load_config(args.config)["classes"]["classes"]
+            classes = load_config(args.config)["classes"]["titles"]
             dist, fg_ratio, qod = compare(args.masks, args.labels, tile, classes)
             if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod:
                 continue
diff --git a/robosat/tools/export.py b/robosat/tools/export.py
index 5887d827..af85d21e 100644
--- a/robosat/tools/export.py
+++ b/robosat/tools/export.py
@@ -32,7 +32,7 @@ def main(args):
         os.environ["CUDA_VISIBLE_DEVICES"] = ""
         # Workaround: PyTorch ONNX, DataParallel with GPU issue, cf https://github.com/pytorch/pytorch/issues/5315
 
-    num_classes = len(config["classes"]["classes"])
+    num_classes = len(config["classes"]["titles"])
     num_channels = 0
     for channel in config["channels"]:
         num_channels += len(channel["bands"])

From bb2dcf8b3adfaeaa60b474e157759c3611c32ca5 Mon Sep 17 00:00:00 2001
From: ocourtin <o@courtin.co>
Date: Fri, 30 Nov 2018 10:14:06 +0100
Subject: [PATCH 96/97] Typo

---
 robosat/tools/tile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/robosat/tools/tile.py b/robosat/tools/tile.py
index c792491c..10a29c7b 100644
--- a/robosat/tools/tile.py
+++ b/robosat/tools/tile.py
@@ -46,7 +46,7 @@ def main(args):
         except:
             sys.exit("Error: Unable to load DataSet config file")
 
-        classes = config["classes"]["title"]
+        classes = config["classes"]["titles"]
         colors = config["classes"]["colors"]
         assert len(classes) == len(colors), "classes and colors coincide"
         assert len(colors) == 2, "only binary models supported right now"

From b28b2c1f7c0a163cefddb9ddc295e9c8ebba9e7b Mon Sep 17 00:00:00 2001
From: Olivier Courtin <o@datapink.com>
Date: Fri, 30 Nov 2018 16:34:51 +0100
Subject: [PATCH 97/97] Add pillow-simd dependancy libs

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b70ed2b8..b5fd0853 100644
--- a/README.md
+++ b/README.md
@@ -87,7 +87,7 @@ The following describes the installation from scratch.
 - Install native system dependencies required for Python 3 bindings
 
 ```bash
-apt-get install build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev
+apt-get install build-essential libboost-python-dev libexpat1-dev zlib1g-dev libbz2-dev libspatialindex-dev libjpeg-turbo8-dev libwebp-dev
 ```
 
 - Use a virtualenv for installing this project locally