From f5c1f58ef62038fce0fbb2a0db1e387f3607710d Mon Sep 17 00:00:00 2001 From: achlus Date: Mon, 13 Jul 2026 08:28:34 -0700 Subject: [PATCH 1/5] Draft coverage script --- emit_main/scripts/create_coverage_json.py | 236 ++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 emit_main/scripts/create_coverage_json.py diff --git a/emit_main/scripts/create_coverage_json.py b/emit_main/scripts/create_coverage_json.py new file mode 100644 index 0000000..518e661 --- /dev/null +++ b/emit_main/scripts/create_coverage_json.py @@ -0,0 +1,236 @@ +""" +A script to compile metrics from various places and record them in a file or database table + +""" + +import argparse +import datetime as dt +import json + +import pandas as pd +from dateutil.relativedelta import relativedelta + +from emit_main.database.database_manager import DatabaseManager + +OBS_VARS = { + "path_length": "Path length (sensor-to-ground in meters)", + "sensor_azimuth": "To-sensor azimuth (0 to 360 degrees CW from N)", + "sensor_zenith": "To-sensor zenith (0 to 90 degrees from zenith)", + "solar_azimuth": "To-sun azimuth (0 to 360 degrees CW from N)", + "solar_zenith": "To-sun zenith (0 to - 90 degrees from zenith)", + "solar_phase": "Solar phase (degrees between to-sensor and to-sun vectors in principal - plane)", + "slope": "Slope (local surface slope as derived from DEM in degrees)", + "aspect": "Aspect (local surface aspect 0 to 360 degrees clockwise from N)", + "cosine_i": "Cosine(i) (apparent local illumination factor based on DEM slope and - aspect and to sun vector)", + "utc_time": "UTC Time (decimal hours for mid-line pixels)", + "earth_sun_dist": "Earth-sun distance (AU)", + } + +STATE_VARS = { + "aot": "Retrieved AOT Median", + "surface_elevation_km": "Retrieved Ele. Median", + "h2o": "Retrieved WV Median" + } + +MASK_VARS = { + "cloudratio_fraction" : "Cloud Cover with ratio", + "cloud_fraction" : "Cloud Fraction Spectf", + "nodata_fraction" : "Screened Onboard Fraction", + } + +def flatten_dict(d, parent_key="", sep="."): + flattened = {} + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + flattened.update(flatten_dict(v, new_key, sep=sep)) + else: + flattened[new_key] = v + return flattened + +def make_feature(record, l1b_v, l2a_v, mask_v): + + record = flatten_dict(record) + st = record.get('start_time') + et = record.get('stop_time') + + fid = record.get('acquisition_id') + + time_string = fid[4:].upper() + + feature = { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": gring_to_polygon(record['gring'])}, + "properties": { + "fid": fid, + "dcid": record.get('associated_dcid'), + "Orbit": record.get('orbit'), + "Orbit Segment": record.get('scene'), + "start_time": st.strftime("%Y-%m-%dT%H:%M:%SZ") if st else None, + "end_time": et.strftime("%Y-%m-%dT%H:%M:%SZ") if et else None, + 'style': {"weight":1, + "opacity":1, + "fillColor": "#0000FF", + "color": "#0000FF"} + }, + } + + for key, name in OBS_VARS.items(): + obs_val = record.get(f'products.l1b.{l1b_v}.obs.band_means.{key}') + if obs_val: + feature["properties"][name] = round(obs_val, 2) + for key, name in STATE_VARS.items(): + state_val = record.get(f'products.l2a.{l2a_v}.state.band_medians.{key}') + if state_val: + feature["properties"][name] = round(state_val, 2) + + for key, name in MASK_VARS.items(): + mask_val = record.get(f'products.mask.{mask_v}.maskTf.{key}') + if mask_val: + feature["properties"][name] = mask_val + + screened = feature["properties"].get("Screened Onboard Fraction", False) + cloud_fraction = feature["properties"].get("Cloud Fraction Spectf", False) + + if screened & cloud_fraction: + feature["properties"]["Cloud Cover"] = screened + cloud_fraction + + on_daac = False + + if record.get(f'products.l1b.{l1b_v}.rdn_ummg'): + l1b_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.0{l1b_v}' + l1b_rad = f'EMIT_L1B_RAD_0{l1b_v}_{time_string}' + l1b_obs = f'EMIT_L1B_OBS_0{l1b_v}_{time_string}' + feature['properties']['L1B Radiance Download'] = f'{l1b_base}/{l1b_rad}/{l1b_rad}.nc' + feature['properties']['L1B Observation Download'] = f'{l1b_base}/{l1b_rad}/{l1b_obs}.nc' + on_daac = True + + if record.get(f'products.l2a.{l2a_v}.rfl_ummg'): + l2a_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.0{l2a_v}' + l2a_rfl = f'EMIT_L2A_RFL_0{l2a_v}_{time_string}' + l2a_rflunc = f'EMIT_L2A_RFLUNCERT_0{l2a_v}_{time_string}' + feature['properties']['L2A Reflectance Download'] = f'{l2a_base}/{l2a_rfl}/{l2a_rfl}.nc' + feature['properties']['L2A Reflectance Uncertainty Download'] = f'{l2a_base}/{l2a_rfl}/{l2a_rflunc}.nc' + on_daac = True + + if record.get(f'products.mask.{mask_v}.maskTf_ummg'): + mask_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2AMASK.0{mask_v}' + l2a_mask = f'EMIT_L2A_MASK_0{mask_v}_{time_string}' + feature['properties']['L2A Mask Download'] = f'{mask_base}/{l2a_mask}/{l2a_mask}.nc' + on_daac = True + + if on_daac == False: + feature['properties']['style'] = {"weight":1,"opacity":1,"fillColor": "#f6c409", "color": "#f6c409"} + + return feature + +def gring_to_polygon(gring): + ring = [list(pt) for pt in gring] + if ring[0] != ring[-1]: + ring.append(ring[0]) + return [ring] + +def open_chunk(n, out_file_chunk_base): + fh = open(f"{out_file_chunk_base}_{n}.json", "w") + fh.write('{\n "type": "FeatureCollection",\n "features": [\n') + return fh + +def close_chunk(fh): + fh.write("\n ]\n}\n") + fh.close() + +def main(): + # Set up args + parser = argparse.ArgumentParser(description="Compile metrics for tracking") + parser.add_argument('out_base', help="Ouput coverage json file") + parser.add_argument("-e", "--env", default="ops", help="Where to run the report") + parser.add_argument("--chunk", action="store_true", default=False, help="Split output into chunked JSON files") + parser.add_argument("--dataframe", action="store_true", default=False, help="Write features to CSV") + parser.add_argument("--chunksize",type = int, default=5000, help="JSON splitting size") + parser.add_argument("--dates", help="Comma separated dates (YYYYMMDD,YYYYMMDD)") + args = parser.parse_args() + + out_file = f'{args.out_base}_pub.json' + out_file_db = f'{args.out_base}_db.csv' + out_file_chunk_base = f'{args.out_base}_pub_chunk' + + config_path = f"/store/emit/{args.env}/repos/emit-main/emit_main/config/{args.env}_sds_config.json" + print(f"Using config_path {config_path}") + + dm = DatabaseManager(config_path) + acq_coll = dm.db.acquisitions + + query = {"gring": {"$exists": True}} + + if args.dates is not None: + parts = args.dates.split(",") + start_date = dt.datetime.strptime(parts[0], "%Y%m%d") + stop_date = dt.datetime.strptime(parts[1], "%Y%m%d") + + query["start_time"] = {"$gte": start_date, "$lt": stop_date} + + # TODO: add dm. + l1b_v = config["product_config"]["prod_versions"]["l1b"] + l2a_v = config["product_config"]["prod_versions"]["l2a"] + mask_v = config["product_config"]["prod_versions"]["mask"] + + projection = { + "gring": 1, + "acquisition_id": 1, + "associated_dcid": 1, + "orbit": 1, + "scene": 1, + "start_time": 1, + "stop_time": 1, + f"products.l1b.{l1b_v}.obs.band_means": 1, + f"products.l2a.{l2a_v}.state.band_medians": 1, + f"products.mask.{mask_v}.maskTf": 1, + f"products.l1b.{l1b_v}.rdn_ummg": 1, + f"products.l2a.{l2a_v}.rfl_ummg": 1, + f"products.mask.{mask_v}.maskTf_ummg": 1, + "_id": 0, + } + + features = [] + chunk_n = 0 + chunk_count = 0 + cf = None + + cursor = acq_coll.find(query, projection, batch_size=1000) + + with open(out_file, "w") as f: + f.write('{\n "type": "FeatureCollection",\n "features": [\n') + first = True + for record in cursor: + feat = make_feature(record, l1b_v, l2a_v, mask_v) + if args.dataframe: + features.append(feat['properties']) + line = " " + json.dumps(feat) + if not first: + f.write(",\n") + f.write(line) + first = False + if args.chunk: + if cf is None: + cf = open_chunk(chunk_n, out_file_chunk_base) + chunk_first = True + if not chunk_first: + cf.write(",\n") + cf.write(line) + chunk_first = False + chunk_count += 1 + if chunk_count >= args.chunk_size: + close_chunk(cf) + cf = None + chunk_n += 1 + chunk_count = 0 + f.write("\n ]\n}\n") + + if cf is not None: + close_chunk(cf) + + if args.dataframe: + pd.DataFrame(features).to_csv(out_file_db, index=False) + +if __name__ == '__main__': + main() From eac241959333b2901bfe82aa545fad8c9deb1622 Mon Sep 17 00:00:00 2001 From: achlus Date: Wed, 15 Jul 2026 11:05:25 -0700 Subject: [PATCH 2/5] Added FID and DAAC indices and sorting --- emit_main/scripts/create_coverage_json.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/emit_main/scripts/create_coverage_json.py b/emit_main/scripts/create_coverage_json.py index 518e661..86449e0 100644 --- a/emit_main/scripts/create_coverage_json.py +++ b/emit_main/scripts/create_coverage_json.py @@ -196,13 +196,22 @@ def main(): chunk_count = 0 cf = None - cursor = acq_coll.find(query, projection, batch_size=1000) + cursor = acq_coll.find(query, projection, batch_size=1000).sort("acquisition_id", 1) + daac_index = 1 + fid_index = 1 with open(out_file, "w") as f: f.write('{\n "type": "FeatureCollection",\n "features": [\n') first = True for record in cursor: feat = make_feature(record, l1b_v, l2a_v, mask_v) + + feat['properties']['FID_index'] = fid_index + feat['properties']['DAAC_index'] = daac_index + fid_index +=1 + if 'L1B Radiance Download' in list(feat['properties'].keys()): + daac_index += 1 + if args.dataframe: features.append(feat['properties']) line = " " + json.dumps(feat) @@ -231,6 +240,6 @@ def main(): if args.dataframe: pd.DataFrame(features).to_csv(out_file_db, index=False) - + if __name__ == '__main__': main() From a08245488f305bf21987ee95f2b7d0db2f1da391 Mon Sep 17 00:00:00 2001 From: achlus Date: Fri, 17 Jul 2026 16:09:45 -0700 Subject: [PATCH 3/5] Key and logic updates --- emit_main/scripts/create_coverage_json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/emit_main/scripts/create_coverage_json.py b/emit_main/scripts/create_coverage_json.py index 86449e0..8cc0d82 100644 --- a/emit_main/scripts/create_coverage_json.py +++ b/emit_main/scripts/create_coverage_json.py @@ -92,12 +92,12 @@ def make_feature(record, l1b_v, l2a_v, mask_v): screened = feature["properties"].get("Screened Onboard Fraction", False) cloud_fraction = feature["properties"].get("Cloud Fraction Spectf", False) - if screened & cloud_fraction: + if screened and cloud_fraction: feature["properties"]["Cloud Cover"] = screened + cloud_fraction on_daac = False - if record.get(f'products.l1b.{l1b_v}.rdn_ummg'): + if record.get(f'products.l1b.{l1b_v}.rdn_ummg.ummg_json_path'): l1b_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL1BRAD.0{l1b_v}' l1b_rad = f'EMIT_L1B_RAD_0{l1b_v}_{time_string}' l1b_obs = f'EMIT_L1B_OBS_0{l1b_v}_{time_string}' @@ -105,7 +105,7 @@ def make_feature(record, l1b_v, l2a_v, mask_v): feature['properties']['L1B Observation Download'] = f'{l1b_base}/{l1b_rad}/{l1b_obs}.nc' on_daac = True - if record.get(f'products.l2a.{l2a_v}.rfl_ummg'): + if record.get(f'products.l2a.{l2a_v}.rfl_ummg.ummg_json_path'): l2a_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2ARFL.0{l2a_v}' l2a_rfl = f'EMIT_L2A_RFL_0{l2a_v}_{time_string}' l2a_rflunc = f'EMIT_L2A_RFLUNCERT_0{l2a_v}_{time_string}' @@ -113,7 +113,7 @@ def make_feature(record, l1b_v, l2a_v, mask_v): feature['properties']['L2A Reflectance Uncertainty Download'] = f'{l2a_base}/{l2a_rfl}/{l2a_rflunc}.nc' on_daac = True - if record.get(f'products.mask.{mask_v}.maskTf_ummg'): + if record.get(f'products.mask.{mask_v}.maskTf_ummg.ummg_json_path'): mask_base = f'https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/EMITL2AMASK.0{mask_v}' l2a_mask = f'EMIT_L2A_MASK_0{mask_v}_{time_string}' feature['properties']['L2A Mask Download'] = f'{mask_base}/{l2a_mask}/{l2a_mask}.nc' From a175681ccf8863bd9fcaf44b0cd9664bc9f86b15 Mon Sep 17 00:00:00 2001 From: achlus Date: Thu, 23 Jul 2026 08:50:50 -0700 Subject: [PATCH 4/5] Description update --- emit_main/scripts/create_coverage_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emit_main/scripts/create_coverage_json.py b/emit_main/scripts/create_coverage_json.py index 8cc0d82..328c51e 100644 --- a/emit_main/scripts/create_coverage_json.py +++ b/emit_main/scripts/create_coverage_json.py @@ -1,5 +1,5 @@ """ -A script to compile metrics from various places and record them in a file or database table +A script to generate coverage geojsons from database """ From 76028daf133a4f058e11d1a8d8f167e31c3e6cab Mon Sep 17 00:00:00 2001 From: achlus Date: Wed, 19 Aug 2026 09:59:55 -0700 Subject: [PATCH 5/5] Added CO2 and updated field name --- emit_main/scripts/create_coverage_json.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/emit_main/scripts/create_coverage_json.py b/emit_main/scripts/create_coverage_json.py index 328c51e..bb62992 100644 --- a/emit_main/scripts/create_coverage_json.py +++ b/emit_main/scripts/create_coverage_json.py @@ -29,11 +29,12 @@ STATE_VARS = { "aot": "Retrieved AOT Median", "surface_elevation_km": "Retrieved Ele. Median", - "h2o": "Retrieved WV Median" + "h2o": "Retrieved WV Median", + "co2": "Retrieved CO2 Median" } MASK_VARS = { - "cloudratio_fraction" : "Cloud Cover with ratio", + "cloudratio_fraction" : "Cloud Ratio and SpecTF Union + Onboard Clouds", "cloud_fraction" : "Cloud Fraction Spectf", "nodata_fraction" : "Screened Onboard Fraction", }