From 342dee7584778954790a0de489115c049487f539 Mon Sep 17 00:00:00 2001 From: Quintin Date: Tue, 17 Mar 2026 10:22:35 -0700 Subject: [PATCH 1/4] Match ncks output format flag to input NetCDF type --- ncpartitioner/response.py | 33 +++++++++++++++++++++++++++++++-- tests/test_response.py | 22 +++++++++++++++++++++- tests/test_sanitize.py | 2 +- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/ncpartitioner/response.py b/ncpartitioner/response.py index 66cdee6..1a2b0dc 100644 --- a/ncpartitioner/response.py +++ b/ncpartitioner/response.py @@ -2,7 +2,6 @@ In cases of DDS and DAS, the file already exists; for data requests the filemust be created first. """ -from posixpath import dirname import subprocess import os from flask import redirect @@ -11,15 +10,45 @@ logger = logging.getLogger(__name__) +def input_filepath(args): + """Resolve the source file path for the current request.""" + if "filepath" in args: + return os.path.join(args["filepath"], f"{args['basename']}.{args['extension']}") + return f"/{args['dirname']}/{args['basename']}.{args['extension']}" + + +def output_format_flag(filepath): + """Select an ncks output format flag that matches the input NetCDF format.""" + input_format = subprocess.check_output( + ["ncdump", "-k", filepath], text=True + ).strip() + + format_flags = { + "classic": "-3", + "64-bit offset": "-6", + "cdf5": "-5", + "netCDF-4 classic model": "-7", + "netCDF-4": "-4", + } + + try: + return format_flags[input_format] + except KeyError as exc: + raise RuntimeError(f"Unsupported netCDF format: {input_format}") from exc + + def slice(args): output_dir = os.getenv("OUTPUT_DIR") thredds_base = os.getenv("THREDDS_HTTP_BASE") + source_filepath = input_filepath(args) logger.info(f"Slicing file") try: + format_flag = output_format_flag(source_filepath) subprocess.run( [ "ncks", + format_flag, "-v", f"{args['variable']}", "-d", @@ -28,7 +57,7 @@ def slice(args): f"lat,{args['lat'][0]},{args['lat'][1]}", "-d", f"lon,{args['lon'][0]},{args['lon'][1]}", - f"/{args['dirname']}/{args['basename']}.{args['extension']}", + source_filepath, os.path.join( output_dir, f"{args['basename']}_{args['timestamp']}.{args['extension']}", diff --git a/tests/test_response.py b/tests/test_response.py index 1df6fd1..e1cee49 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,5 +1,5 @@ import re -from ncpartitioner.response import slice, dds, das +from ncpartitioner.response import slice, dds, das, output_format_flag import os import pytest import subprocess @@ -44,6 +44,26 @@ def test_slice_error(): slice(args) +@pytest.mark.parametrize( + "ncdump_format,expected_flag", + [ + ("classic", "-3"), + ("64-bit offset", "-6"), + ("cdf5", "-5"), + ("netCDF-4 classic model", "-7"), + ("netCDF-4", "-4"), + ], +) +def test_output_format_flag(monkeypatch, ncdump_format, expected_flag): + monkeypatch.setattr( + subprocess, + "check_output", + lambda *args, **kwargs: ncdump_format if kwargs.get("text") else None, + ) + + assert output_format_flag("/tmp/example.nc") == expected_flag + + @pytest.mark.parametrize( "targets,timestamp", [ diff --git a/tests/test_sanitize.py b/tests/test_sanitize.py index 97ebde8..a514755 100644 --- a/tests/test_sanitize.py +++ b/tests/test_sanitize.py @@ -92,7 +92,7 @@ def test_check_targets(targets, valid, error): ) def test_check_ranges(args, valid, error): args["basename"] = "tasmax" - dirname = "tests/data/" + dirname = "tests/data" args["dirname"] = dirname args["extension"] = "nc" From ee4c8f997c16200515e68512162170d430a796c6 Mon Sep 17 00:00:00 2001 From: Quintin Date: Tue, 17 Mar 2026 11:24:59 -0700 Subject: [PATCH 2/4] Install netcdf-bin in python CI --- .github/workflows/python-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 09fe46b..9930834 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -4,7 +4,6 @@ on: push jobs: test: - runs-on: ubuntu-22.04 strategy: matrix: @@ -20,23 +19,24 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 - with: + with: python-version: ${{ matrix.python-version }} - name: Install OS Dependencies run: | sudo apt update sudo apt install python3-dev - sudo apt-get install nco + sudo apt-get install nco + sudo apt-get install netcdf-bin sudo apt-get install curl - name: Install Poetry run: | pip install poetry==2.2 - + - name: Install project run: | poetry install - name: Test with pytest - run: poetry run pytest \ No newline at end of file + run: poetry run pytest From 011e66602cbf6bede57c6e0aea551aca5f1461bc Mon Sep 17 00:00:00 2001 From: Quintin Date: Tue, 17 Mar 2026 11:42:17 -0700 Subject: [PATCH 3/4] Add netcdf-bin to Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 88295d0..90cff2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y \ nco \ - curl + curl \ + netcdf-bin COPY . /app WORKDIR /app From 9b63ad5585401b259dc73f744c59a786e85af3e8 Mon Sep 17 00:00:00 2001 From: Quintin Date: Tue, 17 Mar 2026 13:23:35 -0700 Subject: [PATCH 4/4] Revert, and use ncks -4 flag for all inputs --- .github/workflows/python-ci.yml | 1 - Dockerfile | 3 +-- ncpartitioner/response.py | 23 +---------------------- tests/test_response.py | 22 +--------------------- 4 files changed, 3 insertions(+), 46 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 9930834..aace047 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -27,7 +27,6 @@ jobs: sudo apt update sudo apt install python3-dev sudo apt-get install nco - sudo apt-get install netcdf-bin sudo apt-get install curl - name: Install Poetry diff --git a/Dockerfile b/Dockerfile index 90cff2f..38de56b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,7 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y \ nco \ - curl \ - netcdf-bin + curl COPY . /app WORKDIR /app diff --git a/ncpartitioner/response.py b/ncpartitioner/response.py index 1a2b0dc..3c05258 100644 --- a/ncpartitioner/response.py +++ b/ncpartitioner/response.py @@ -17,26 +17,6 @@ def input_filepath(args): return f"/{args['dirname']}/{args['basename']}.{args['extension']}" -def output_format_flag(filepath): - """Select an ncks output format flag that matches the input NetCDF format.""" - input_format = subprocess.check_output( - ["ncdump", "-k", filepath], text=True - ).strip() - - format_flags = { - "classic": "-3", - "64-bit offset": "-6", - "cdf5": "-5", - "netCDF-4 classic model": "-7", - "netCDF-4": "-4", - } - - try: - return format_flags[input_format] - except KeyError as exc: - raise RuntimeError(f"Unsupported netCDF format: {input_format}") from exc - - def slice(args): output_dir = os.getenv("OUTPUT_DIR") thredds_base = os.getenv("THREDDS_HTTP_BASE") @@ -44,11 +24,10 @@ def slice(args): logger.info(f"Slicing file") try: - format_flag = output_format_flag(source_filepath) subprocess.run( [ "ncks", - format_flag, + "-4", "-v", f"{args['variable']}", "-d", diff --git a/tests/test_response.py b/tests/test_response.py index e1cee49..1df6fd1 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,5 +1,5 @@ import re -from ncpartitioner.response import slice, dds, das, output_format_flag +from ncpartitioner.response import slice, dds, das import os import pytest import subprocess @@ -44,26 +44,6 @@ def test_slice_error(): slice(args) -@pytest.mark.parametrize( - "ncdump_format,expected_flag", - [ - ("classic", "-3"), - ("64-bit offset", "-6"), - ("cdf5", "-5"), - ("netCDF-4 classic model", "-7"), - ("netCDF-4", "-4"), - ], -) -def test_output_format_flag(monkeypatch, ncdump_format, expected_flag): - monkeypatch.setattr( - subprocess, - "check_output", - lambda *args, **kwargs: ncdump_format if kwargs.get("text") else None, - ) - - assert output_format_flag("/tmp/example.nc") == expected_flag - - @pytest.mark.parametrize( "targets,timestamp", [