From 7f34ea3580ba68e74ab0f97d24d3ac6fd2e968a2 Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 16 Nov 2020 21:13:35 -0500 Subject: [PATCH 1/5] added detections-vehicles to covid-api --- covid_api/db/static/datasets/__init__.py | 6 ++- covid_api/db/static/datasets/agriculture.json | 3 +- covid_api/db/static/datasets/co2-diff.json | 3 +- covid_api/db/static/datasets/co2.json | 3 +- .../db/static/datasets/detections-plane.json | 3 +- .../db/static/datasets/detections-ship.json | 3 +- .../static/datasets/detections-vehicles.json | 41 +++++++++++++++++++ .../db/static/datasets/nightlights-hd.json | 3 +- .../db/static/datasets/nightlights-viirs.json | 3 +- covid_api/db/static/datasets/no2.json | 3 +- .../db/static/datasets/water-chlorophyll.json | 3 +- covid_api/db/static/datasets/water-spm.json | 3 +- 12 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 covid_api/db/static/datasets/detections-vehicles.json diff --git a/covid_api/db/static/datasets/__init__.py b/covid_api/db/static/datasets/__init__.py index 92b1201..2952ae9 100644 --- a/covid_api/db/static/datasets/__init__.py +++ b/covid_api/db/static/datasets/__init__.py @@ -133,7 +133,11 @@ def _prep_output(self, output_datasets: dict, api_url: str): tile.replace("{api_url}", api_url) for tile in dataset.compare.source.tiles ] - if dataset.id in ["detections-ship", "detections-plane"]: + if dataset.id in [ + "detections-ship", + "detections-plane", + "detections-vehicles", + ]: dataset.source = GeoJsonSource( type=dataset.source.type, data=dataset.source.tiles[0] ) diff --git a/covid_api/db/static/datasets/agriculture.json b/covid_api/db/static/datasets/agriculture.json index e149122..37013fb 100644 --- a/covid_api/db/static/datasets/agriculture.json +++ b/covid_api/db/static/datasets/agriculture.json @@ -24,7 +24,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "enabled": false, "swatch": { diff --git a/covid_api/db/static/datasets/co2-diff.json b/covid_api/db/static/datasets/co2-diff.json index 56a4623..5932c5a 100644 --- a/covid_api/db/static/datasets/co2-diff.json +++ b/covid_api/db/static/datasets/co2-diff.json @@ -24,7 +24,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "enabled": false, "swatch": { diff --git a/covid_api/db/static/datasets/co2.json b/covid_api/db/static/datasets/co2.json index 41aeb85..9a79d83 100644 --- a/covid_api/db/static/datasets/co2.json +++ b/covid_api/db/static/datasets/co2.json @@ -23,7 +23,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "enabled": false, "compare": { diff --git a/covid_api/db/static/datasets/detections-plane.json b/covid_api/db/static/datasets/detections-plane.json index 9831d30..f74b2b8 100644 --- a/covid_api/db/static/datasets/detections-plane.json +++ b/covid_api/db/static/datasets/detections-plane.json @@ -29,7 +29,8 @@ "detection-multi", "water-chlorophyll", "water-spm", - "detections-ship" + "detections-ship", + "detections-vehicles" ], "enabled": false, "swatch": { diff --git a/covid_api/db/static/datasets/detections-ship.json b/covid_api/db/static/datasets/detections-ship.json index 3637b60..4d80781 100644 --- a/covid_api/db/static/datasets/detections-ship.json +++ b/covid_api/db/static/datasets/detections-ship.json @@ -29,7 +29,8 @@ "detection-multi", "water-chlorophyll", "water-spm", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "enabled": false, "swatch": { diff --git a/covid_api/db/static/datasets/detections-vehicles.json b/covid_api/db/static/datasets/detections-vehicles.json new file mode 100644 index 0000000..91043b0 --- /dev/null +++ b/covid_api/db/static/datasets/detections-vehicles.json @@ -0,0 +1,41 @@ +{ + "id": "detections-vehicles", + "name": "Vehicle", + "type": "inference-timeseries", + "s3_location": "detections-vehicles", + "is_periodic": false, + "time_unit": "day", + "source": { + "type": "geojson", + "tiles": [ + "{api_url}/detections/vehicles/{spotlightId}/{date}.geojson" + ] + }, + "background_source": { + "type": "raster", + "tiles": [ + "{api_url}/planet/{z}/{x}/{y}?date={date}&site={spotlightId}" + ] + }, + "exclusive_with": [ + "agriculture", + "no2", + "co2-diff", + "co2", + "gibs-population", + "car-count", + "nightlights-viirs", + "nightlights-hd", + "detection-multi", + "water-chlorophyll", + "water-spm", + "detections-ship", + "detections-plane" + ], + "enabled": false, + "swatch": { + "color": "#C0C0C0", + "name": "Grey" + }, + "info": "Vehicles detected each day in PlanetScope imagery are shown in red." +} \ No newline at end of file diff --git a/covid_api/db/static/datasets/nightlights-hd.json b/covid_api/db/static/datasets/nightlights-hd.json index a44d25f..ceeb37a 100644 --- a/covid_api/db/static/datasets/nightlights-hd.json +++ b/covid_api/db/static/datasets/nightlights-hd.json @@ -23,7 +23,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "swatch": { "color": "#C0C0C0", diff --git a/covid_api/db/static/datasets/nightlights-viirs.json b/covid_api/db/static/datasets/nightlights-viirs.json index 51e0d67..3dd8c8e 100644 --- a/covid_api/db/static/datasets/nightlights-viirs.json +++ b/covid_api/db/static/datasets/nightlights-viirs.json @@ -23,7 +23,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "swatch": { "color": "#C0C0C0", diff --git a/covid_api/db/static/datasets/no2.json b/covid_api/db/static/datasets/no2.json index 387e8ea..7fa2f83 100644 --- a/covid_api/db/static/datasets/no2.json +++ b/covid_api/db/static/datasets/no2.json @@ -23,7 +23,8 @@ "water-chlorophyll", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "enabled": true, "compare": { diff --git a/covid_api/db/static/datasets/water-chlorophyll.json b/covid_api/db/static/datasets/water-chlorophyll.json index 146d09f..511a01b 100644 --- a/covid_api/db/static/datasets/water-chlorophyll.json +++ b/covid_api/db/static/datasets/water-chlorophyll.json @@ -23,7 +23,8 @@ "detection-multi", "water-spm", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "swatch": { "color": "#154F8D", diff --git a/covid_api/db/static/datasets/water-spm.json b/covid_api/db/static/datasets/water-spm.json index c4052b7..3540c36 100644 --- a/covid_api/db/static/datasets/water-spm.json +++ b/covid_api/db/static/datasets/water-spm.json @@ -23,7 +23,8 @@ "detection-multi", "water-chlorophyll", "detections-ship", - "detections-plane" + "detections-plane", + "detections-vehicles" ], "swatch": { "color": "#154F8D", From fbabcb145b10301817f373c9cf143e02908cb74f Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 16 Nov 2020 21:37:01 -0500 Subject: [PATCH 2/5] added `detections-vehicles` to `detections` API endpoint --- covid_api/api/api_v1/endpoints/detections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covid_api/api/api_v1/endpoints/detections.py b/covid_api/api/api_v1/endpoints/detections.py index 075980a..669b29e 100644 --- a/covid_api/api/api_v1/endpoints/detections.py +++ b/covid_api/api/api_v1/endpoints/detections.py @@ -12,7 +12,7 @@ router = APIRouter() # TODO: unhardcoded types and dates -MLTypes = Enum("MLTypes", [(ml, ml) for ml in ["ship", "multiple", "plane"]]) # type: ignore +MLTypes = Enum("MLTypes", [(ml, ml) for ml in ["ship", "multiple", "plane", "vehicles"]]) # type: ignore @router.get( From 96f29d96728909946e929482ec584aafe1d1ec4b Mon Sep 17 00:00:00 2001 From: Olaf Veerman Date: Wed, 18 Nov 2020 10:08:56 +0000 Subject: [PATCH 3/5] Update description of detections --- covid_api/db/static/datasets/detections-plane.json | 2 +- covid_api/db/static/datasets/detections-ship.json | 2 +- covid_api/db/static/datasets/detections-vehicles.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/covid_api/db/static/datasets/detections-plane.json b/covid_api/db/static/datasets/detections-plane.json index f74b2b8..17ffa5e 100644 --- a/covid_api/db/static/datasets/detections-plane.json +++ b/covid_api/db/static/datasets/detections-plane.json @@ -37,5 +37,5 @@ "color": "#C0C0C0", "name": "Grey" }, - "info": "Planes detected each day in PlanetScope imagery are shown in red." + "info": "Planes detected each day in PlanetScope imagery are shown in orange." } \ No newline at end of file diff --git a/covid_api/db/static/datasets/detections-ship.json b/covid_api/db/static/datasets/detections-ship.json index 4d80781..bfaf832 100644 --- a/covid_api/db/static/datasets/detections-ship.json +++ b/covid_api/db/static/datasets/detections-ship.json @@ -37,5 +37,5 @@ "color": "#C0C0C0", "name": "Grey" }, - "info": "Ships detected each day in PlanetScope imagery are shown in red." + "info": "Ships detected each day in PlanetScope imagery are shown in orange." } \ No newline at end of file diff --git a/covid_api/db/static/datasets/detections-vehicles.json b/covid_api/db/static/datasets/detections-vehicles.json index 91043b0..c8bcb46 100644 --- a/covid_api/db/static/datasets/detections-vehicles.json +++ b/covid_api/db/static/datasets/detections-vehicles.json @@ -37,5 +37,5 @@ "color": "#C0C0C0", "name": "Grey" }, - "info": "Vehicles detected each day in PlanetScope imagery are shown in red." + "info": "Vehicles detected each day in PlanetScope imagery are shown in orange." } \ No newline at end of file From f5a55ee3df3abb72697139ce28d116ad8e051cd5 Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Fri, 20 Nov 2020 09:47:48 -0500 Subject: [PATCH 4/5] added retrieval of scene ids for detections-vehicles dataset --- covid_api/api/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/covid_api/api/utils.py b/covid_api/api/utils.py index 79c86c4..f688505 100644 --- a/covid_api/api/utils.py +++ b/covid_api/api/utils.py @@ -729,8 +729,16 @@ def site_date_to_scenes(site: str, date: str): ) ship_site_date_lines = ship_site_date_to_scenes_csv.decode("utf-8").split("\n") + vehicles_site_date_to_scenes_csv = s3_get( + INDICATOR_BUCKET, "detections-vehicles/detection_scenes.csv" + ) + vehicles_site_date_lines = vehicles_site_date_to_scenes_csv.decode("utf-8").split( + "\n" + ) + reader = list(csv.DictReader(plane_site_date_lines)) reader.extend(list(csv.DictReader(ship_site_date_lines))) + reader.extend(list(csv.DictReader(vehicles_site_date_lines))) site_date_to_scenes_dict: dict = {} From 64ea5dd7f3d3c33eeb651514181654b8c74a78ab Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Fri, 20 Nov 2020 16:17:55 -0500 Subject: [PATCH 5/5] added Skysat background images for detections-vehicles dataset and update background location in metadata file --- covid_api/api/utils.py | 12 +++--------- covid_api/db/static/datasets/detections-plane.json | 2 +- covid_api/db/static/datasets/detections-ship.json | 2 +- .../db/static/datasets/detections-vehicles.json | 4 ++-- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/covid_api/api/utils.py b/covid_api/api/utils.py index f688505..177164d 100644 --- a/covid_api/api/utils.py +++ b/covid_api/api/utils.py @@ -729,16 +729,8 @@ def site_date_to_scenes(site: str, date: str): ) ship_site_date_lines = ship_site_date_to_scenes_csv.decode("utf-8").split("\n") - vehicles_site_date_to_scenes_csv = s3_get( - INDICATOR_BUCKET, "detections-vehicles/detection_scenes.csv" - ) - vehicles_site_date_lines = vehicles_site_date_to_scenes_csv.decode("utf-8").split( - "\n" - ) - reader = list(csv.DictReader(plane_site_date_lines)) reader.extend(list(csv.DictReader(ship_site_date_lines))) - reader.extend(list(csv.DictReader(vehicles_site_date_lines))) site_date_to_scenes_dict: dict = {} @@ -748,4 +740,6 @@ def site_date_to_scenes(site: str, date: str): json.loads(row["scene_id"].replace("'", '"')) ) - return site_date_to_scenes_dict[f"{site}-{date}"] + # deduplicate scene list (in case multiple datasets contains the same + # scene id) + return list(set(site_date_to_scenes_dict[f"{site}-{date}"])) diff --git a/covid_api/db/static/datasets/detections-plane.json b/covid_api/db/static/datasets/detections-plane.json index f74b2b8..fbd1585 100644 --- a/covid_api/db/static/datasets/detections-plane.json +++ b/covid_api/db/static/datasets/detections-plane.json @@ -8,7 +8,7 @@ "source": { "type": "geojson", "tiles": [ - "{api_url}/detections/plane/{spotlightId}/{date}.geojson" + "{api_url}/detections-plane/{spotlightId}/{date}.geojson" ] }, "background_source": { diff --git a/covid_api/db/static/datasets/detections-ship.json b/covid_api/db/static/datasets/detections-ship.json index 4d80781..ef5b477 100644 --- a/covid_api/db/static/datasets/detections-ship.json +++ b/covid_api/db/static/datasets/detections-ship.json @@ -8,7 +8,7 @@ "source": { "type": "geojson", "tiles": [ - "{api_url}/detections/ship/{spotlightId}/{date}.geojson" + "{api_url}/detections-ship/{spotlightId}/{date}.geojson" ] }, "background_source": { diff --git a/covid_api/db/static/datasets/detections-vehicles.json b/covid_api/db/static/datasets/detections-vehicles.json index 91043b0..fcfdbee 100644 --- a/covid_api/db/static/datasets/detections-vehicles.json +++ b/covid_api/db/static/datasets/detections-vehicles.json @@ -8,13 +8,13 @@ "source": { "type": "geojson", "tiles": [ - "{api_url}/detections/vehicles/{spotlightId}/{date}.geojson" + "{api_url}/detections-vehicles/{spotlightId}/{date}.geojson" ] }, "background_source": { "type": "raster", "tiles": [ - "{api_url}/planet/{z}/{x}/{y}?date={date}&site={spotlightId}" + "{api_url}/{z}/{x}/{y}@1x?url=s3://covid-eo-data/detections-vehicles/background/{spotlightId}/{date}.tif" ] }, "exclusive_with": [