(0.3.0) Refactor bounding box to Metadata.region and add Column

.github/workflows/ci.yml

@@ -218,7 +218,7 @@ jobs:
               using Pkg;
               Pkg.test(; coverage=true,
                         julia_args=["--check-bounds=yes", "--compiled-modules=yes", "-O0"],
-                        test_args=["--verbose", "test_cds_downloading", "test_downloading"])
+                        test_args=["--verbose", "test_cds_downloading", "test_jra55_ecco_en4_etopo_downloading", "test_glorys_downloading"])
           '
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v5

.github/workflows/docs.yml

@@ -25,6 +25,10 @@ env:
   NUMERICAL_EARTH_LABEL_BUILD_ALL_EXAMPLES: 'build all examples'
   CDSAPI_URL: "https://cds.climate.copernicus.eu/api"
   CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}
+  COPERNICUS_USERNAME: ${{ secrets.COPERNICUSMARINE_SERVICE_USERNAME }}
+  COPERNICUS_PASSWORD: ${{ secrets.COPERNICUSMARINE_SERVICE_PASSWORD }}
+  ECCO_USERNAME: ${{ secrets.ECCO_USERNAME }}
+  ECCO_WEBDAV_PASSWORD: ${{ secrets.ECCO_WEBDAV_PASSWORD }}
 
 jobs:
   build-docs:
@@ -63,10 +67,6 @@ jobs:
         run: julia --project=docs --color=yes -e 'using Pkg; Pkg.instantiate(verbose=true)'
       - name: Build documentation
         env:
-          ECCO_USERNAME: ${{ secrets.ECCO_USERNAME }}
-          ECCO_WEBDAV_PASSWORD: ${{ secrets.ECCO_WEBDAV_PASSWORD }}
-          COPERNICUS_USERNAME: ${{ secrets.COPERNICUS_SERVICE_USERNAME }}
-          COPERNICUS_PASSWORD: ${{ secrets.COPERNICUS_USERNAME_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JULIA_DEBUG: Documenter
           JULIA_SSL_NO_VERIFY: "**"

.gitignore

@@ -64,3 +64,6 @@ docs/src/literated/
 .CondaPkg
 CondaPkg.toml
 !docs/CondaPkg.toml
+
+# claude
+.claude

Project.toml

@@ -1,7 +1,7 @@
 name = "NumericalEarth"
 uuid = "904d977b-046a-4731-8b86-9235c0d1ef02"
 license = "MIT"
-version = "0.2.3"
+version = "0.3.0"
 authors = ["NumericalEarth contributors"]
 
 [deps]

docs/CondaPkg.toml

@@ -1,4 +1,6 @@
-# Pin Python to < 3.14 to avoid the free-threaded build (cp314t)
-# which is ABI-incompatible with PythonCall.
+
+[deps.copernicusmarine]
+channel = "conda-forge"
+
 [deps.python]
 version = ">=3.10,<3.14"

docs/Project.toml

@@ -6,6 +6,7 @@ CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 ClimaSeaIce = "6ba0ff68-24e6-4315-936c-2e99227c95a4"
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
+CopernicusMarine = "cd43e856-93a3-40c8-bc9e-6146cdce14fa"
 DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"

docs/src/Metadata/metadata_overview.md

@@ -64,9 +64,11 @@ The key ingredients stored in a [`Metadata`](@ref) or [`Metadatum`](@ref) object
 - the variable name (for example `:temperature` or `:u_velocity`);
 - the dataset (such as `EN4Monthly`, `ECCO2Daily`, or `GLORYSMonthly`);
 - the temporal coverage: either a single timestamp (`Metadatum`) or a range/vector of dates (`Metadata`);
-- an optional [`BoundingBox`](@ref NumericalEarth.DataWrangling.BoundingBox) describing regional subsets in
-  longitude, latitude, or depth;
-- the on-disk `dir`ectory where the dataset are be cached.
+- an optional `region` describing the spatial extent — either a
+  [`BoundingBox`](@ref NumericalEarth.DataWrangling.BoundingBox) for a rectangular sub-domain,
+  a [`Column`](@ref NumericalEarth.DataWrangling.Column) for a single horizontal location, or
+  `nothing` for the full global domain (see [Regions, locations, and FieldTimeSeries](@ref));
+- the on-disk `dir`ectory where the dataset files are cached.
 
 This bookkeeping lets downstream utilities (for example `set!` or `FieldTimeSeries`) request exactly the
 slices of data they need, and it keeps track of where those slices live so we do not redownload

examples/download_glorys_data.jl

@@ -17,20 +17,20 @@ grid = LatitudeLongitudeGrid(arch;
                              latitude  = (35, 55),
                              longitude = (200, 220))
 
-bounding_box = NumericalEarth.DataWrangling.BoundingBox(longitude=(200, 220), latitude=(35, 55))
+region = NumericalEarth.DataWrangling.BoundingBox(longitude=(200, 220), latitude=(35, 55))
 
 # dataset = NumericalEarth.DataWrangling.Copernicus.GLORYSStatic()
-# static_meta = NumericalEarth.DataWrangling.Metadatum(:depth; dataset, bounding_box)
+# static_meta = NumericalEarth.DataWrangling.Metadatum(:depth; dataset, region)
 # coords_path = NumericalEarth.DataWrangling.download_dataset(static_meta)
 # @info "Downloaded coordinates data to $coords_path"
 
-# T_ecco = NumericalEarth.DataWrangling.ECCOMetadatum(:temperature; dataset, bounding_box)
+# T_ecco = NumericalEarth.DataWrangling.ECCOMetadatum(:temperature; dataset, region)
 # T_en4_meta = NumericalEarth.DataWrangling.EN4Metadatum(:temperature)
 # T_en4_path = NumericalEarth.DataWrangling.download_dataset(T_en4_meta)
 # T_en4 = Field(T_en4_meta)
 
 dataset = NumericalEarth.DataWrangling.Copernicus.GLORYSDaily()
-T_meta = NumericalEarth.DataWrangling.Metadatum(:temperature; dataset, bounding_box)
+T_meta = NumericalEarth.DataWrangling.Metadatum(:temperature; dataset, region)
 T_path = NumericalEarth.DataWrangling.download_dataset(T_meta)
 @info "Downloaded temperature data to $T_path"
 T = Field(T_meta, inpainting=nothing)

examples/single_column_os_papa_simulation.jl

@@ -14,6 +14,7 @@
 # pkg"add Oceananigans, NumericalEarth, CairoMakie"
 # ```
 
+using CopernicusMarine
 using NumericalEarth
 using Oceananigans
 using Oceananigans: prognostic_fields
@@ -29,7 +30,7 @@ using Printf
 
 # Ocean station papa location
 location_name = "ocean_station_papa"
-λ★, φ★ = 35.1, 50.1
+λ★, φ★ = -144.9, 50.1
 
 grid = RectilinearGrid(size = 200,
                        x = λ★,
@@ -48,10 +49,14 @@ ocean = ocean_simulation(grid; Δt=10minutes, coriolis=FPlane(latitude = φ★))
 
 ocean.model
 
-# We set initial conditions from ECCO4:
+# We set initial conditions from GLORYS, using a `Column` region so that
+# only the single water column at `(λ★, φ★)` is downloaded from the
+# Copernicus Marine Service.
 
-set!(ocean.model, T=Metadatum(:temperature, dataset=ECCO4Monthly()),
-                  S=Metadatum(:salinity, dataset=ECCO4Monthly()))
+col = Column(λ★, φ★; interpolation=Nearest())
+
+set!(ocean.model, T=Metadatum(:temperature, dataset=GLORYSMonthly(), region=col),
+                  S=Metadatum(:salinity,    dataset=GLORYSMonthly(), region=col))
 
 # # A prescribed atmosphere based on JRA55 re-analysis
 #

ext/NumericalEarthCDSAPIExt.jl

@@ -77,7 +77,7 @@ function download_dataset(meta::ERA5Metadatum; skip_existing=true)
     )
 
     # Add area constraint from bounding box
-    area = build_era5_area(meta.bounding_box)
+    area = build_era5_area(meta.region)
     if !isnothing(area)
         request["area"] = area
     end

ext/NumericalEarthCopernicusMarineExt.jl

@@ -46,11 +46,12 @@ function download_dataset(meta::GLORYSMetadatum;
         (; start_datetime, end_datetime)
     end
 
-    lon_kw = longitude_bounds_kw(meta.bounding_box)
-    lat_kw = latitude_bounds_kw(meta.bounding_box)
-    z_kw = depth_bounds_kw(meta.bounding_box)
+    lon_kw = longitude_bounds_kw(meta.region)
+    lat_kw = latitude_bounds_kw(meta.region)
+    z_kw = depth_bounds_kw(meta.region)
+    selection_method = coordinates_selection_method(meta.region)
 
-    kw = (; coordinates_selection_method = "outside",
+    kw = (; coordinates_selection_method = selection_method,
           skip_existing,
           dataset_id,
           variables,
@@ -76,12 +77,38 @@ end
 longitude_bounds_kw(::Nothing) = NamedTuple()
 latitude_bounds_kw(::Nothing) = NamedTuple()
 depth_bounds_kw(::Nothing) = NamedTuple()
+coordinates_selection_method(::Nothing) = "outside"
 
 const BBOX = NumericalEarth.DataWrangling.BoundingBox
+const COL  = NumericalEarth.DataWrangling.Column
+const LIN  = NumericalEarth.DataWrangling.Linear
+const NR   = NumericalEarth.DataWrangling.Nearest
+
+longitude_bounds_kw(bbox::BBOX) = longitude_bounds_kw(bbox.longitude)
+latitude_bounds_kw(bbox::BBOX) = latitude_bounds_kw(bbox.latitude)
+depth_bounds_kw(bbox::BBOX) = depth_bounds_kw(bbox.z)
+coordinates_selection_method(::BBOX) = "outside"
+
+# Column with Nearest interpolation: download the single nearest point
+longitude_bounds_kw(col::COL{<:Any, <:Any, <:Any, NR}) = (; minimum_longitude = col.longitude, maximum_longitude = col.longitude)
+latitude_bounds_kw(col::COL{<:Any, <:Any, <:Any, NR})  = (; minimum_latitude = col.latitude, maximum_latitude = col.latitude)
+depth_bounds_kw(col::COL) = depth_bounds_kw(col.z)
+coordinates_selection_method(::COL{<:Any, <:Any, <:Any, NR}) = "nearest"
+
+# Column with Linear interpolation: expand by a small margin for interpolation
+longitude_bounds_kw(col::COL{<:Any, <:Any, <:Any, LIN}) = expand_longitude(col.longitude)
+latitude_bounds_kw(col::COL{<:Any, <:Any, <:Any, LIN})  = expand_latitude(col.latitude)
+coordinates_selection_method(::COL{<:Any, <:Any, <:Any, LIN}) = "outside"
+
+function expand_longitude(lon)
+    ε = 1/6  # slightly more than one GLORYS grid cell (1/12°)
+    return (; minimum_longitude = lon - ε, maximum_longitude = lon + ε)
+end
 
-longitude_bounds_kw(bounding_box::BBOX) = longitude_bounds_kw(bounding_box.longitude)
-latitude_bounds_kw(bounding_box::BBOX) = latitude_bounds_kw(bounding_box.latitude)
-depth_bounds_kw(bounding_box::BBOX) = depth_bounds_kw(bounding_box.z)
+function expand_latitude(lat)
+    ε = 1/6
+    return (; minimum_latitude = lat - ε, maximum_latitude = lat + ε)
+end
 
 function longitude_bounds_kw(longitude)
     minimum_longitude = longitude[1]

src/DataWrangling/DataWrangling.jl

@@ -5,11 +5,13 @@ restoring, or validation.
 module DataWrangling
 
 export Metadata, Metadatum, DatewiseFilename, ECCOMetadatum, EN4Metadatum, all_dates, first_date, last_date
+export BoundingBox, Column, Linear, Nearest
 export WOAClimatology, WOAAnnual, WOAMonthly
 export metadata_time_step, metadata_epoch
 export LinearlyTaperedPolarMask
 export DatasetRestoring, SurfaceFluxRestoring
 export ERA5Hourly, ERA5Monthly
+export native_grid
 
 using Oceananigans
 using Downloads
@@ -189,6 +191,7 @@ function z_interfaces end
 function longitude_interfaces end
 function latitude_interfaces end
 function reversed_vertical_axis end
+reversed_latitude_axis(dataset) = false
 function native_grid end
 function binary_data_grid end
 function binary_data_size end

src/DataWrangling/ECCO/ECCO.jl

@@ -21,27 +21,33 @@ using NumericalEarth.DataWrangling:
     netrc_downloader,
     NearestNeighborInpainting,
     BoundingBox,
+    Column,
     metadata_path,
     GramPerKilogramMinus35,
     MicromolePerLiter,
     Metadata,
     Metadatum,
     download_progress,
-    InverseSign
+    InverseSign,
+    native_grid,
+    location,
+    compute_mask,
+    inpaint_mask!,
+    set_metadata_field!,
+    extract_column!
 
 using KernelAbstractions: @kernel, @index
 
 using Dates: year, month, day
 
-import Oceananigans: location
-
 import NumericalEarth.DataWrangling:
     default_download_directory,
     all_dates,
     metadata_filename,
     download_dataset,
     conversion_units,
     dataset_variable_name,
+    dataset_location,
     metaprefix,
     longitude_interfaces,
     latitude_interfaces,
@@ -247,7 +253,7 @@ end
 metaprefix(::ECCOMetadata) = "ECCOMetadata"
 
 # File name generation specific to each dataset
-function metadata_filename(::ECCO4Monthly, name, date, bounding_box)
+function metadata_filename(::ECCO4Monthly, name, date, region)
     shortname = ECCO4_dataset_variable_names[name]
     yearstr   = string(Dates.year(date))
     monthstr  = string(Dates.month(date), pad=2)
@@ -260,7 +266,7 @@ ecco2_is_three_dimensional(name) =
     name == :u_velocity ||
     name == :v_velocity
 
-function metadata_filename(dataset::Union{ECCO2Daily, ECCO2Monthly}, name, date, bounding_box)
+function metadata_filename(dataset::Union{ECCO2Daily, ECCO2Monthly}, name, date, region)
     shortname = ECCO2_dataset_variable_names[name]
     yearstr   = string(Dates.year(date))
     monthstr  = string(Dates.month(date), pad=2)
@@ -278,7 +284,7 @@ end
 dataset_variable_name(data::Metadata{<:ECCO2Daily})   = ECCO2_dataset_variable_names[data.name]
 dataset_variable_name(data::Metadata{<:ECCO2Monthly}) = ECCO2_dataset_variable_names[data.name]
 dataset_variable_name(data::Metadata{<:ECCO4Monthly}) = ECCO4_dataset_variable_names[data.name]
-location(data::ECCOMetadata) = ECCO_location[data.name]
+dataset_location(::ECCODataset, name) = ECCO_location[name]
 
 is_three_dimensional(data::ECCOMetadata) =
     data.name == :temperature ||
@@ -365,4 +371,37 @@ inpainted_metadata_path(metadata::ECCOMetadatum) = joinpath(metadata.dir, inpain
 
 include("ECCO_atmosphere.jl")
 
+#####
+##### Column Field for ECCO datasets (which always download globally)
+#####
+
+using Oceananigans.BoundaryConditions: fill_halo_regions!
+
+const ECCOColumnMetadatum = Metadatum{<:ECCODataset, <:Any, <:Column}
+
+function Oceananigans.Fields.Field(metadata::ECCOColumnMetadatum, arch=CPU();
+                                   inpainting = default_inpainting(metadata),
+                                   mask = nothing,
+                                   halo = (3, 3, 3),
+                                   cache_inpainted_data = true)
+
+    download_dataset(metadata)
+    column_grid = native_grid(metadata, arch; halo)
+
+    # Build a full-grid Field without a region to load the global data
+    global_metadatum = Metadatum(metadata.name;
+                                 dataset = metadata.dataset,
+                                 date = metadata.dates)
+
+    intermediate_field = Field(global_metadatum, arch; inpainting, mask, halo, cache_inpainted_data)
+    fill_halo_regions!(intermediate_field)
+
+    # Extract the column
+    _, _, LZ = location(metadata)
+    column_field = Field{Nothing, Nothing, LZ}(column_grid)
+    extract_column!(column_field, intermediate_field, metadata.region)
+
+    return column_field
+end
+
 end # Module

src/DataWrangling/ECCO/ECCO_darwin.jl

@@ -26,14 +26,14 @@ all_dates(dataset::ECCO2DarwinMonthly, name) = metadata_epoch(dataset) : Month(1
 
 # File name generation specific to each Dataset dataset
 """
-    metadata_filename(dataset, name, date, bounding_box)
+    metadata_filename(dataset, name, date, region)
 
 Generate the filename for a given ECCO Darwin dataset and date.
 
 The filename is constructed using the dataset variable name, and the iteration number is calculated
 from the date and epoch.
 """
-function metadata_filename(dataset::Union{ECCO2DarwinMonthly, ECCO4DarwinMonthly}, name, date, bounding_box)
+function metadata_filename(dataset::Union{ECCO2DarwinMonthly, ECCO4DarwinMonthly}, name, date, region)
     shortname = ECCO_darwin_dataset_variable_names[name]
 
     reference_date = metadata_epoch(dataset)
@@ -52,8 +52,6 @@ default_mask_value(::ECCO2DarwinMonthly) = 0
 
 dataset_variable_name(data::Metadata{<:Union{ECCO2DarwinMonthly,ECCO4DarwinMonthly}}) = ECCO_darwin_dataset_variable_names[data.name]
 
-location(::Metadata{<:Union{ECCO2DarwinMonthly, ECCO4DarwinMonthly}}) = (Center, Center, Center)
-
 variable_is_three_dimensional(::Metadata{<:Union{ECCO2DarwinMonthly, ECCO4DarwinMonthly}}) = true
 
 ECCO_darwin_dataset_variable_names = Dict(

src/DataWrangling/EN4/EN4.jl

@@ -50,7 +50,6 @@ import NumericalEarth.DataWrangling:
     inpainted_metadata_path,
     available_variables
 
-import Oceananigans.Fields: location
 
 download_EN4_cache::String = ""
 function __init__()
@@ -160,15 +159,14 @@ metaprefix(::EN4Metadatum) = "EN4Metadatum"
 
 # Note, EN4 files contain all variables, so the filenames do not
 # depend on name.
-function metadata_filename(::EN4Monthly, name, date, bounding_box)
+function metadata_filename(::EN4Monthly, name, date, region)
     yearstr  = string(Dates.year(date))
     monthstr = string(Dates.month(date), pad=2)
     return "EN.4.2.2.f.analysis.g10." * yearstr * lpad(string(monthstr), 2, '0') * ".nc"
 end
 
 # Convenience functions
 dataset_variable_name(data::EN4Metadata) = EN4_dataset_variable_names[data.name]
-location(::EN4Metadata) = (Center, Center, Center)
 is_three_dimensional(::EN4Metadata) = true
 
 ## This function is explicitly for the downloader to check if the zip file/extracted file exists,