From 331865298334e1340456dec09ed5546a40e8f9f7 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Wed, 8 Jul 2026 13:46:09 -0700 Subject: [PATCH 1/7] Terracotta try catch --- src/nuthatch/backends/terracotta.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/nuthatch/backends/terracotta.py b/src/nuthatch/backends/terracotta.py index 0db6ed0..eaef1a9 100644 --- a/src/nuthatch/backends/terracotta.py +++ b/src/nuthatch/backends/terracotta.py @@ -193,7 +193,6 @@ def write(self, ds, upsert=False, primary_keys=None): ds = ds.rio.reproject('EPSG:3857', resampling=self.resampling, nodata=np.nan) ds.rio.write_crs("epsg:3857", inplace=True) - # Insert the parameters. with self.driver.connect(verify=False): if self.time_dim in ds.dims: for t in ds.time: @@ -201,13 +200,24 @@ def write(self, ds, upsert=False, primary_keys=None): sub_ds = ds.sel(time=t) sub_ds = sub_ds.reset_coords('time', drop=True) - # Assume the cache_key is now a folder and make sub tifs under it + # Skip time slices with no valid data - terracotta can't + # compute metadata for an all-nodata raster + if not any(bool(sub_ds[v].notnull().any()) for v in sub_ds.data_vars): + logger.warning(f"Skipping {self.cache_key}_{t.values}: no valid (non-NaN) data.") + continue + sub_cache_key = self.cache_key + '_' + str(t.values) sub_path = os.path.join(self.path, str(t.values) + '.tif') - sub_override_path = os.path.join(self.override_path, str(t.values) + '.tif') - - self.write_individual_raster(self.driver, sub_ds, sub_path, sub_cache_key, sub_override_path) + sub_override_path = os.path.join(self.override_path, str(t.values) + '.tif') + try: + self.write_individual_raster(self.driver, sub_ds, sub_path, sub_cache_key, sub_override_path) + except Exception: + logger.exception(f"Failed to write raster {sub_cache_key}; continuing with remaining times.") + continue else: + if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): + logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") + return ds path = os.path.join(self.path, '_.tif') override_path = os.path.join(self.override_path, '_.tif') self.write_individual_raster(self.driver, ds, path, self.cache_key, override_path) From 690a69f7d317e56e0b007c3eec5b2b8505bef331 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Wed, 8 Jul 2026 14:21:56 -0700 Subject: [PATCH 2/7] Added testing --- tests/backends/test_terracotta.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/backends/test_terracotta.py b/tests/backends/test_terracotta.py index c8491b0..72e693e 100644 --- a/tests/backends/test_terracotta.py +++ b/tests/backends/test_terracotta.py @@ -59,6 +59,27 @@ def time_latlon_array(): return ds +@cache(cache_args=[]) +def time_latlon_array_with_null_chunks(): + """Generate a timeseries dataset where some time slices are all null.""" + lats = np.arange(-90, 90, 0.25) + lons = np.arange(-180, 180, 0.25) + times = pd.date_range("2001-01-01", "2001-01-03", freq='1d') + + data_shape = (len(lats), len(lons), len(times)) + empty_data = np.ones(data_shape) * np.random.randint(1000000) + empty_data[:, :, :2] = np.nan + + ds = xr.Dataset( + data_vars= { + 'data': (['lat', 'lon', 'time'], empty_data), + }, + coords={"lat": lats, "lon": lons, "time": times}, + ) + + return ds + + def test_terracotta(): # cache the array ds = simple_latlon_array(backend='zarr') @@ -73,3 +94,10 @@ def test_terracotta(): ds = time_latlon_array(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') ds = time_latlon_array(backend='terracotta') assert len(ds) == 5 + + +def test_terracotta_all_null_time_chunks(): + time_latlon_array_with_null_chunks(backend='zarr') + time_latlon_array_with_null_chunks(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') + ds = time_latlon_array_with_null_chunks(backend='terracotta') + assert len(ds) == 1 From 7b98c37113e236d07f819f1078bb72ce71b4fa95 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Wed, 8 Jul 2026 15:46:58 -0700 Subject: [PATCH 3/7] Fixed terracotta keying and tests --- src/nuthatch/backends/terracotta.py | 47 ++++++++++++++++++++--------- tests/backends/test_terracotta.py | 26 ++++++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/nuthatch/backends/terracotta.py b/src/nuthatch/backends/terracotta.py index eaef1a9..a4d5a4f 100644 --- a/src/nuthatch/backends/terracotta.py +++ b/src/nuthatch/backends/terracotta.py @@ -3,7 +3,7 @@ import terracotta as tc import sqlalchemy import xarray as xr -import rioxarray # Must import for .rio to work # noqa: F401 +import rioxarray # Must import for .rio to work # noqa: F401 import numpy as np from rasterio.io import MemoryFile from rasterio.enums import Resampling @@ -13,6 +13,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) + def base360_to_base180(lons): """Converts a list of longitudes from base 360 to base 180. @@ -41,7 +42,6 @@ def base180_to_base360(lons): return np.array(val) - def is_wrapped(lons): """Check if the longitudes are wrapped. @@ -187,6 +187,12 @@ def write(self, ds, upsert=False, primary_keys=None): ds = ds.transpose(self.time_dim, 'y', 'x') else: ds = ds.transpose('y', 'x') + if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): + logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") + self.fs.makedirs(self.path, exist_ok=True) + with self.fs.open(os.path.join(self.path, '.null'), 'wb') as f: + f.write(b'') + return ds # Adapt the CRS ds.rio.write_crs("epsg:4326", inplace=True) @@ -215,9 +221,6 @@ def write(self, ds, upsert=False, primary_keys=None): logger.exception(f"Failed to write raster {sub_cache_key}; continuing with remaining times.") continue else: - if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): - logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") - return ds path = os.path.join(self.path, '_.tif') override_path = os.path.join(self.override_path, '_.tif') self.write_individual_raster(self.driver, ds, path, self.cache_key, override_path) @@ -233,29 +236,44 @@ def write_individual_raster(self, driver, ds, path, cache_key, override_path): shutil.copyfileobj(mem_dst, f_out) driver.insert({'key': cache_key.replace('/', '_')}, mem_dst, - override_path=override_path, skip_metadata=False) + override_path=override_path, skip_metadata=False) logger.info(f"Inserted {cache_key.replace('/', '_')} into the terracotta database.") def upsert(self, data, upsert_keys=None): raise NotImplementedError("Terracotta does not support upsert.") + def _matches_cache_key(self, dataset_key): + # Terracotta keys are either the 2D base key or a time slice: + # time_latlon_array_ (from cache_key "time_latlon_array/") + # time_latlon_array__2001-01-01 (base key + "_" + timestamp) + # + # A broad SQL LIKE on base_key + "%" also matches unrelated caches whose + # names share a prefix (e.g. time_latlon_array_with_null_chunks__...). + # We can't fix that in SQL with LIKE alone because "_" is a wildcard there. + base_key = self.cache_key.replace('/', '_') + if dataset_key == base_key: + return True + return dataset_key.startswith(base_key + '_') + def read(self, engine): datasets_table = sqlalchemy.Table("datasets", self.driver.meta_store.sqla_metadata, autoload_with=self.driver.meta_store.sqla_engine) + base_key = self.cache_key.replace('/', '_') + # Broad SQL filter, then exact prefix check in _matches_cache_key. stmt = ( datasets_table.select() - .where(datasets_table.c['key'].like(self.cache_key.replace('/', '_') + '%')) + .where(datasets_table.c['key'].like(base_key + '%')) ) with self.driver.meta_store.connect() as conn: result = conn.execute(stmt).all() - datasets = [row[0] for row in result] - ret = [] - for dataset in datasets: - ret.append({dataset: self.driver.get_metadata({'key' : dataset})}) + for row in result: + if not self._matches_cache_key(row[0]): + continue + ret.append({row[0]: self.driver.get_metadata({'key': row[0]})}) return ret @@ -265,17 +283,16 @@ def delete(self): datasets_table = sqlalchemy.Table("datasets", self.driver.meta_store.sqla_metadata, autoload_with=self.driver.meta_store.sqla_engine) + base_key = self.cache_key.replace('/', '_') stmt = ( datasets_table.select() - .where(datasets_table.c['key'].like(self.cache_key.replace('/', '_') + '%')) + .where(datasets_table.c['key'].like(base_key + '%')) ) - with self.driver.meta_store.connect() as conn: result = conn.execute(stmt).all() - datasets = [row[0] for row in result] + datasets = [row[0] for row in result if self._matches_cache_key(row[0])] for dataset in datasets: logger.info(f"Deleting datasets {datasets} from terracotta.") self.driver.delete({'key': dataset}) - diff --git a/tests/backends/test_terracotta.py b/tests/backends/test_terracotta.py index 72e693e..256a6d1 100644 --- a/tests/backends/test_terracotta.py +++ b/tests/backends/test_terracotta.py @@ -80,6 +80,22 @@ def time_latlon_array_with_null_chunks(): return ds +@cache(cache_args=[]) +def all_null_latlon_array(): + """Generate a 2D lat/lon dataset with no valid data.""" + lats = np.arange(-90, 90, 0.25) + lons = np.arange(-180, 180, 0.25) + + ds = xr.Dataset( + data_vars= { + 'data': (['lat', 'lon'], np.full((len(lats), len(lons)), np.nan)), + }, + coords={"lat": lats, "lon": lons}, + ) + + return ds + + def test_terracotta(): # cache the array ds = simple_latlon_array(backend='zarr') @@ -89,6 +105,9 @@ def test_terracotta(): assert len(ds) == 1 # cache the array + time_latlon_array_with_null_chunks(backend='zarr') + time_latlon_array_with_null_chunks(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') + ds = time_latlon_array(backend='zarr') # Verify that write? ds = time_latlon_array(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') @@ -96,6 +115,13 @@ def test_terracotta(): assert len(ds) == 5 +def test_terracotta_all_null_no_time(): + all_null_latlon_array(backend='zarr') + all_null_latlon_array(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') + ds = all_null_latlon_array(backend='terracotta') + assert len(ds) == 0 + + def test_terracotta_all_null_time_chunks(): time_latlon_array_with_null_chunks(backend='zarr') time_latlon_array_with_null_chunks(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') From 23bd41236053732c1448865f66c6b4c1db837675 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Wed, 8 Jul 2026 15:51:47 -0700 Subject: [PATCH 4/7] Clean up big changes --- src/nuthatch/backends/terracotta.py | 39 +++++++++++------------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/nuthatch/backends/terracotta.py b/src/nuthatch/backends/terracotta.py index a4d5a4f..29c797f 100644 --- a/src/nuthatch/backends/terracotta.py +++ b/src/nuthatch/backends/terracotta.py @@ -187,12 +187,6 @@ def write(self, ds, upsert=False, primary_keys=None): ds = ds.transpose(self.time_dim, 'y', 'x') else: ds = ds.transpose('y', 'x') - if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): - logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") - self.fs.makedirs(self.path, exist_ok=True) - with self.fs.open(os.path.join(self.path, '.null'), 'wb') as f: - f.write(b'') - return ds # Adapt the CRS ds.rio.write_crs("epsg:4326", inplace=True) @@ -221,6 +215,12 @@ def write(self, ds, upsert=False, primary_keys=None): logger.exception(f"Failed to write raster {sub_cache_key}; continuing with remaining times.") continue else: + if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): + logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") + # Marker file so FileBackend.exists() sees this cache (see test_terracotta_all_null_no_time). + with self.fs.open(os.path.join(self.path, '.null'), 'wb') as f: + f.write(b'') + return ds path = os.path.join(self.path, '_.tif') override_path = os.path.join(self.override_path, '_.tif') self.write_individual_raster(self.driver, ds, path, self.cache_key, override_path) @@ -243,24 +243,10 @@ def write_individual_raster(self, driver, ds, path, cache_key, override_path): def upsert(self, data, upsert_keys=None): raise NotImplementedError("Terracotta does not support upsert.") - def _matches_cache_key(self, dataset_key): - # Terracotta keys are either the 2D base key or a time slice: - # time_latlon_array_ (from cache_key "time_latlon_array/") - # time_latlon_array__2001-01-01 (base key + "_" + timestamp) - # - # A broad SQL LIKE on base_key + "%" also matches unrelated caches whose - # names share a prefix (e.g. time_latlon_array_with_null_chunks__...). - # We can't fix that in SQL with LIKE alone because "_" is a wildcard there. - base_key = self.cache_key.replace('/', '_') - if dataset_key == base_key: - return True - return dataset_key.startswith(base_key + '_') - def read(self, engine): datasets_table = sqlalchemy.Table("datasets", self.driver.meta_store.sqla_metadata, autoload_with=self.driver.meta_store.sqla_engine) base_key = self.cache_key.replace('/', '_') - # Broad SQL filter, then exact prefix check in _matches_cache_key. stmt = ( datasets_table.select() .where(datasets_table.c['key'].like(base_key + '%')) @@ -269,11 +255,15 @@ def read(self, engine): with self.driver.meta_store.connect() as conn: result = conn.execute(stmt).all() + # LIKE also matches longer cache keys with the same prefix; filter in Python + # because "_" is a wildcard in SQL LIKE. + time_slice_prefix = base_key + '_' ret = [] for row in result: - if not self._matches_cache_key(row[0]): + key = row[0] + if key != base_key and not key.startswith(time_slice_prefix): continue - ret.append({row[0]: self.driver.get_metadata({'key': row[0]})}) + ret.append({key: self.driver.get_metadata({'key': key})}) return ret @@ -283,16 +273,15 @@ def delete(self): datasets_table = sqlalchemy.Table("datasets", self.driver.meta_store.sqla_metadata, autoload_with=self.driver.meta_store.sqla_engine) - base_key = self.cache_key.replace('/', '_') stmt = ( datasets_table.select() - .where(datasets_table.c['key'].like(base_key + '%')) + .where(datasets_table.c['key'].like(self.cache_key.replace('/', '_') + '%')) ) with self.driver.meta_store.connect() as conn: result = conn.execute(stmt).all() - datasets = [row[0] for row in result if self._matches_cache_key(row[0])] + datasets = [row[0] for row in result] for dataset in datasets: logger.info(f"Deleting datasets {datasets} from terracotta.") self.driver.delete({'key': dataset}) From d25c95b234ad35ee85cdc065f96e8539c90f3183 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Wed, 8 Jul 2026 15:53:19 -0700 Subject: [PATCH 5/7] Linting --- src/nuthatch/backends/terracotta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nuthatch/backends/terracotta.py b/src/nuthatch/backends/terracotta.py index 29c797f..a2ab518 100644 --- a/src/nuthatch/backends/terracotta.py +++ b/src/nuthatch/backends/terracotta.py @@ -217,7 +217,7 @@ def write(self, ds, upsert=False, primary_keys=None): else: if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") - # Marker file so FileBackend.exists() sees this cache (see test_terracotta_all_null_no_time). + # Marker file so FileBackend.exists() sees this cache with self.fs.open(os.path.join(self.path, '.null'), 'wb') as f: f.write(b'') return ds From a77bc6b2cf2064aa9dd4f44960cd5a21ba3516db Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Thu, 9 Jul 2026 12:03:35 -0700 Subject: [PATCH 6/7] Removed else handling of nulls, just let terracotta fail naturally --- src/nuthatch/backends/terracotta.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/nuthatch/backends/terracotta.py b/src/nuthatch/backends/terracotta.py index a2ab518..8999072 100644 --- a/src/nuthatch/backends/terracotta.py +++ b/src/nuthatch/backends/terracotta.py @@ -215,12 +215,6 @@ def write(self, ds, upsert=False, primary_keys=None): logger.exception(f"Failed to write raster {sub_cache_key}; continuing with remaining times.") continue else: - if not any(bool(ds[v].notnull().any()) for v in ds.data_vars): - logger.warning(f"Skipping {self.cache_key}: no valid (non-NaN) data.") - # Marker file so FileBackend.exists() sees this cache - with self.fs.open(os.path.join(self.path, '.null'), 'wb') as f: - f.write(b'') - return ds path = os.path.join(self.path, '_.tif') override_path = os.path.join(self.override_path, '_.tif') self.write_individual_raster(self.driver, ds, path, self.cache_key, override_path) From db09930e25cf80288a4eb96ae4deeedd444974a9 Mon Sep 17 00:00:00 2001 From: Genevieve Flaspohler Date: Thu, 9 Jul 2026 12:05:24 -0700 Subject: [PATCH 7/7] Updated testing --- tests/backends/test_terracotta.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/tests/backends/test_terracotta.py b/tests/backends/test_terracotta.py index 256a6d1..4511a08 100644 --- a/tests/backends/test_terracotta.py +++ b/tests/backends/test_terracotta.py @@ -24,12 +24,11 @@ def simple_latlon_array(): lats = np.arange(-90, 90, 0.25) lons = np.arange(-180, 180, 0.25) - data_shape = (len(lats), len(lons)) empty_data = np.ones(data_shape) * np.random.randint(1000000) ds = xr.Dataset( - data_vars= { + data_vars={ 'data': (['lat', 'lon'], empty_data), }, coords={"lat": lats, "lon": lons}, @@ -45,12 +44,11 @@ def time_latlon_array(): lons = np.arange(-180, 180, 0.25) times = pd.date_range("2001-01-01", "2001-01-05", freq='1d') - data_shape = (len(lats), len(lons), len(times)) empty_data = np.ones(data_shape) * np.random.randint(1000000) ds = xr.Dataset( - data_vars= { + data_vars={ 'data': (['lat', 'lon', 'time'], empty_data), }, coords={"lat": lats, "lon": lons, "time": times}, @@ -71,7 +69,7 @@ def time_latlon_array_with_null_chunks(): empty_data[:, :, :2] = np.nan ds = xr.Dataset( - data_vars= { + data_vars={ 'data': (['lat', 'lon', 'time'], empty_data), }, coords={"lat": lats, "lon": lons, "time": times}, @@ -80,22 +78,6 @@ def time_latlon_array_with_null_chunks(): return ds -@cache(cache_args=[]) -def all_null_latlon_array(): - """Generate a 2D lat/lon dataset with no valid data.""" - lats = np.arange(-90, 90, 0.25) - lons = np.arange(-180, 180, 0.25) - - ds = xr.Dataset( - data_vars= { - 'data': (['lat', 'lon'], np.full((len(lats), len(lons)), np.nan)), - }, - coords={"lat": lats, "lon": lons}, - ) - - return ds - - def test_terracotta(): # cache the array ds = simple_latlon_array(backend='zarr') @@ -115,13 +97,6 @@ def test_terracotta(): assert len(ds) == 5 -def test_terracotta_all_null_no_time(): - all_null_latlon_array(backend='zarr') - all_null_latlon_array(backend='zarr', storage_backend='terracotta', cache_mode='overwrite') - ds = all_null_latlon_array(backend='terracotta') - assert len(ds) == 0 - - def test_terracotta_all_null_time_chunks(): time_latlon_array_with_null_chunks(backend='zarr') time_latlon_array_with_null_chunks(backend='zarr', storage_backend='terracotta', cache_mode='overwrite')