From 2d5439964a09e5ae3742371107a95e7111065126 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:27:37 +0100 Subject: [PATCH 01/80] #32: Basic prototype callback addition --- lib/ants/fileformats/ancil/__init__.py | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index fb1546d..76c7fc7 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -21,6 +21,7 @@ cube.attributes['grid_staggering']. """ +import glob import ants import iris import mule @@ -50,12 +51,16 @@ def __init__(self, grid_staggering): def __call__(self, cube, field, filename): """ - ANTS callback to add grid staggering and maintain pseudo level order. + ANTS callback to add grid staggering, maintain pseudo level order. Used as a callback when loading fields files for all ants.io.load operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` etc). + This callback will load additional metadata files with the naming convention: + filename. and append the contents of those files to the cube + attributes. + Parameters ---------- cube : :class:`iris.cube.Cube` @@ -68,8 +73,32 @@ def __call__(self, cube, field, filename): """ cube.attributes["grid_staggering"] = self.grid_staggering[filename] + metadata_files = glob.glob(filename+".*") + if metadata_files: + self._retrieve_metadata(metadata_files, cube) super(_CallbackUM, self).__call__(cube, field, filename) + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + for metadata_file in metadata_files: + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + cube.attributes[attribute_name] = metadata + class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): @@ -345,8 +374,13 @@ def load_cubes(*args, **kwargs): :func:`iris.fileformats.um.load_cubes` """ + #change this function grid_staggering = _fetch_grid_staggering_from_file(args[0]) args, kwargs = pp._add_callback(_CallbackUM(grid_staggering), *args, **kwargs) + # here + #get the filepath - args[0]? search for files of the same name +. + #load in contents as attribute dictionary - in callback? + # add to cube return iris.fileformats.um.load_cubes(*args, **kwargs) From 116ddb20866c4b730740c5065ee271f983080243 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 12 Nov 2025 14:23:25 +0000 Subject: [PATCH 02/80] #32: update pytest tempdir config --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ab7ed41..e5cbaf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ skip = ".git" # Configure pytest. [tool.pytest.ini_options] addopts = "--durations=5" +tmp_path_retention_policy = "none" filterwarnings = [ # During unittesting only, treat all warnings as errors by default: 'error', From ab3f3cca5d127d604f239bfa1f701790f11faf52 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:12:41 +0000 Subject: [PATCH 03/80] #32: Working prototype --- lib/ants/io/load.py | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 6aa48d9..46d732b 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -57,6 +57,7 @@ """ import copy import warnings +import glob from contextlib import contextmanager from functools import wraps @@ -354,6 +355,23 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) + user_callback = None + if len(args)>1: + user_callback = args[1] + else: + if 'callback' in kwargs: + user_callback = kwargs.pop('callback') + if 'ignore_metadata_files' in kwargs: + ignore_metadata_files = kwargs.pop('ignore_metadata_files') + #Do the handling for each way a user callback can be passed in through iris + if ignore_metadata_files == False: + args, kwargs = _add_metadata_file_callback(user_callback, args, kwargs) + #_create_callabck_metadata + #do the create callback function and append to callbacks + # (copy pp callbak if user callback exists then use the existing + # pp callback code to append that to the ants callback) + else: + args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -368,6 +386,75 @@ def load_function(*args, **kwargs): return load_function +def _add_metadata_file_callback(user_callback, *args, **kwargs): + """ + have a seperate function for loading metadata + if user callback is not none + do an update the same way as in the pp file + copy pp callback to be the same and add user one to run after this + + """ + args, kwargs = _add_callback(_CallbackMetadata(), *args, **kwargs) + return args, kwargs + +def _add_callback(callback, *args, **kwargs): + """ + Adds both the ants callback and the user provided callback (if any) to the + load. + + """ + args = list(args) + if len(args) == 1: + kwargs['callback'] = callback + else: + args[2] = callback + args = tuple(args) + return args, kwargs + +class _CallbackMetadata(object): + """Callback for collecting metadata from sidecar files. + + This callback will load additional metadata files with the naming convention: + filename. and append the contents of those files to the cube + attributes. + """ + def __init__(self, user_callback): + self._user_callback = user_callback + + def __call__(self, cube, field, filename): + """ + + """ + metadata_filenames = ''.join([filename, ".*"]) + metadata_files = glob.glob(metadata_filenames) + if metadata_files != []: + self._retrieve_metadata(metadata_files, cube) + if self._user_callback is not None: + self._user_callback(cube, field, filename) + + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + for metadata_file in metadata_files: + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + cube.attributes[attribute_name] = metadata + with open('written_license.txt', 'a')as file: + file.write(''.join(metadata)) + def load_cube(*args, **kwargs): """ From 9af1590eaa1f9d1537fe8efc56c95ec9088d3fb0 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 13 Nov 2025 17:00:45 +0000 Subject: [PATCH 04/80] #32: Working prototype with improved logic --- lib/ants/io/load.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 46d732b..605cac0 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -355,22 +355,19 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) - user_callback = None - if len(args)>1: - user_callback = args[1] - else: - if 'callback' in kwargs: - user_callback = kwargs.pop('callback') + ignore_metadata_files = False if 'ignore_metadata_files' in kwargs: ignore_metadata_files = kwargs.pop('ignore_metadata_files') - #Do the handling for each way a user callback can be passed in through iris - if ignore_metadata_files == False: - args, kwargs = _add_metadata_file_callback(user_callback, args, kwargs) - #_create_callabck_metadata - #do the create callback function and append to callbacks - # (copy pp callbak if user callback exists then use the existing - # pp callback code to append that to the ants callback) - else: + if ignore_metadata_files == False: + #Do the handling for each way a user callback can be passed in through iris + user_callback = None + if len(args)>1: + user_callback = args[1] + else: + if 'callback' in kwargs: + user_callback = kwargs.pop('callback') + print("first args: ", args) + print("first kwargs: ", kwargs) args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): @@ -386,17 +383,6 @@ def load_function(*args, **kwargs): return load_function -def _add_metadata_file_callback(user_callback, *args, **kwargs): - """ - have a seperate function for loading metadata - if user callback is not none - do an update the same way as in the pp file - copy pp callback to be the same and add user one to run after this - - """ - args, kwargs = _add_callback(_CallbackMetadata(), *args, **kwargs) - return args, kwargs - def _add_callback(callback, *args, **kwargs): """ Adds both the ants callback and the user provided callback (if any) to the @@ -404,11 +390,17 @@ def _add_callback(callback, *args, **kwargs): """ args = list(args) + print(args) + print("len args: ", len(args)) if len(args) == 1: kwargs['callback'] = callback + elif len(args) == 2: + args[1] = callback else: args[2] = callback args = tuple(args) + print("with first callback args: ", args) + print("with first callback kwargs: ", kwargs) return args, kwargs class _CallbackMetadata(object): @@ -425,6 +417,7 @@ def __call__(self, cube, field, filename): """ """ + print("callback has been added") metadata_filenames = ''.join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: From dff558e3ad81a0058e82963cd2870158bd84029a Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 09:35:14 +0000 Subject: [PATCH 05/80] #32: Update argument handler to correctly pass in user callback --- lib/ants/io/load.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 605cac0..84c3339 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -361,8 +361,8 @@ def load_function(*args, **kwargs): if ignore_metadata_files == False: #Do the handling for each way a user callback can be passed in through iris user_callback = None - if len(args)>1: - user_callback = args[1] + if len(args)==3: + user_callback = args[2] else: if 'callback' in kwargs: user_callback = kwargs.pop('callback') @@ -394,9 +394,7 @@ def _add_callback(callback, *args, **kwargs): print("len args: ", len(args)) if len(args) == 1: kwargs['callback'] = callback - elif len(args) == 2: - args[1] = callback - else: + elif len(args) == 3: args[2] = callback args = tuple(args) print("with first callback args: ", args) @@ -423,6 +421,7 @@ def __call__(self, cube, field, filename): if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: + print("should have added user callback") self._user_callback(cube, field, filename) def _retrieve_metadata(self, metadata_files, cube): From de3d650d10c4c3cba312ba7b47eb98962d52c9ba Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 10:02:33 +0000 Subject: [PATCH 06/80] #32: revert changes to the ancil loading --- lib/ants/fileformats/ancil/__init__.py | 34 -------------------------- 1 file changed, 34 deletions(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index 76c7fc7..abc31bc 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -57,10 +57,6 @@ def __call__(self, cube, field, filename): operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` etc). - This callback will load additional metadata files with the naming convention: - filename. and append the contents of those files to the cube - attributes. - Parameters ---------- cube : :class:`iris.cube.Cube` @@ -73,33 +69,8 @@ def __call__(self, cube, field, filename): """ cube.attributes["grid_staggering"] = self.grid_staggering[filename] - metadata_files = glob.glob(filename+".*") - if metadata_files: - self._retrieve_metadata(metadata_files, cube) super(_CallbackUM, self).__call__(cube, field, filename) - def _retrieve_metadata(self, metadata_files, cube): - """ - Reads in the contents of each file and adds an attribute to the cube with the - name of the file. - - Parameters - ---------- - metadata_files: list - The list of filenames that contain metadata for the cube. - cube : :class:`iris.cube.Cube` - The cube being loaded. - - """ - for metadata_file in metadata_files: - open_file = open(metadata_file, "r") - metadata = open_file.readlines() - open_file.close() - file_name_splits = str(metadata_file).split(".") - attribute_name = file_name_splits[-1] - cube.attributes[attribute_name] = metadata - - class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): self.ppfield = ppfield @@ -374,13 +345,8 @@ def load_cubes(*args, **kwargs): :func:`iris.fileformats.um.load_cubes` """ - #change this function grid_staggering = _fetch_grid_staggering_from_file(args[0]) args, kwargs = pp._add_callback(_CallbackUM(grid_staggering), *args, **kwargs) - # here - #get the filepath - args[0]? search for files of the same name +. - #load in contents as attribute dictionary - in callback? - # add to cube return iris.fileformats.um.load_cubes(*args, **kwargs) From 877a88ebcb1c1ead0ef658a691de91fab14068c8 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:52:05 +0000 Subject: [PATCH 07/80] #32: Add etsts for new loading functionality --- lib/ants/fileformats/ancil/__init__.py | 1 + lib/ants/io/load.py | 43 ++++++---- .../tests/fileformats/pp/test_CallbackPP.py | 5 +- .../tests/io/load/test_CallbackMetadata.py | 83 +++++++++++++++++++ 4 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 lib/ants/tests/io/load/test_CallbackMetadata.py diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index abc31bc..056b6cf 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -71,6 +71,7 @@ def __call__(self, cube, field, filename): cube.attributes["grid_staggering"] = self.grid_staggering[filename] super(_CallbackUM, self).__call__(cube, field, filename) + class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): self.ppfield = ppfield diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 84c3339..6d760e2 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -356,19 +356,27 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False - if 'ignore_metadata_files' in kwargs: - ignore_metadata_files = kwargs.pop('ignore_metadata_files') + if "ignore_metadata_files" in kwargs: + ignore_metadata_files = kwargs.pop("ignore_metadata_files") + print( + "ignore metadata files: ", + ignore_metadata_files, + type(ignore_metadata_files), + ) if ignore_metadata_files == False: - #Do the handling for each way a user callback can be passed in through iris + print("doing the thing") + # Do the handling for each way a user callback can be passed in through iris user_callback = None - if len(args)==3: + if len(args) == 3: user_callback = args[2] else: - if 'callback' in kwargs: - user_callback = kwargs.pop('callback') + if "callback" in kwargs: + user_callback = kwargs.pop("callback") print("first args: ", args) print("first kwargs: ", kwargs) - args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) + args, kwargs = _add_callback( + _CallbackMetadata(user_callback), *args, **kwargs + ) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -383,40 +391,39 @@ def load_function(*args, **kwargs): return load_function + def _add_callback(callback, *args, **kwargs): """ Adds both the ants callback and the user provided callback (if any) to the load. - """ args = list(args) - print(args) - print("len args: ", len(args)) if len(args) == 1: - kwargs['callback'] = callback + kwargs["callback"] = callback elif len(args) == 3: args[2] = callback args = tuple(args) - print("with first callback args: ", args) - print("with first callback kwargs: ", kwargs) return args, kwargs + class _CallbackMetadata(object): """Callback for collecting metadata from sidecar files. - + This callback will load additional metadata files with the naming convention: filename. and append the contents of those files to the cube attributes. """ + def __init__(self, user_callback): self._user_callback = user_callback def __call__(self, cube, field, filename): """ - + The method that runs when iris runs the callback. Collects the filenames and + will run the user callback """ print("callback has been added") - metadata_filenames = ''.join([filename, ".*"]) + metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) @@ -444,8 +451,8 @@ def _retrieve_metadata(self, metadata_files, cube): file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] cube.attributes[attribute_name] = metadata - with open('written_license.txt', 'a')as file: - file.write(''.join(metadata)) + with open("written_license.txt", "a") as file: + file.write("".join(metadata)) def load_cube(*args, **kwargs): diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index 628299d..a93d964 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -4,6 +4,9 @@ # See LICENSE.txt in the root of the repository for full licensing details. import unittest.mock as mock +import tempfile +import os +import pytest import ants.tests import iris @@ -18,7 +21,7 @@ def setUp(self): def call(self): callback = CallbackPP() callback.append_user_callback(self.user_callback) - callback(self.cube, mock.sentinel.field, mock.sentinel.filename) + callback(self.cube, mock.sentinel.field, "filename") def test__freeze_pseudo_level_is_called_if_pseudo_level_present(self): pseudo_level = iris.coords.DimCoord([0], long_name="pseudo_level") diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py new file mode 100644 index 0000000..50a3c5b --- /dev/null +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -0,0 +1,83 @@ +""" +Includes tests for end-to-end functionality of CallbackMetadata as well as calling the +class directly. +""" + +import ants.io.load +import iris +import pytest +import unittest.mock as mock + + +def test_metadata_files_added_to_attributes(tmp_path): + """Tests that metadata files are found and added to the cube's attributes.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + # How the text should look while stored in an array + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_test_cube.attributes["license"] == loaded_license + + +def test_no_metadata_loaded(tmp_path): + """Tests that metadata files are not loaded when the option is turned off.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube( + temporary_cube_path, ignore_metadata_files=True + ) + with pytest.raises(KeyError): + loaded_test_cube.attributes["license"] + + +def test_user_callback_added(): + """Test that on ititialisation, the user's function will be set.""" + + def user_callback(cube, field, filename): + print("a user's callback, passed in") + + class_instance = ants.io.load._CallbackMetadata(user_callback) + assert class_instance._user_callback == user_callback + + +def test_args_parsed_correctly_with_kwargs(tmp_path): + """Tests that when passed a callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, callback=mock_callback) + mock_callback.assert_called() + + +def test_args_parsed_correctly_with_positional_args(tmp_path): + """Tests that when passed a callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, None, mock_callback) + mock_callback.assert_called() From fa39ac105e20ddc8bab7e7405a6921541703dda2 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:53:50 +0000 Subject: [PATCH 08/80] #32: Revert changes to ancil/__init__.py --- lib/ants/fileformats/ancil/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index 056b6cf..fb1546d 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -21,7 +21,6 @@ cube.attributes['grid_staggering']. """ -import glob import ants import iris import mule @@ -51,7 +50,7 @@ def __init__(self, grid_staggering): def __call__(self, cube, field, filename): """ - ANTS callback to add grid staggering, maintain pseudo level order. + ANTS callback to add grid staggering and maintain pseudo level order. Used as a callback when loading fields files for all ants.io.load operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` From 4c354b716c13d8b84d8781da724eb73b8dcde5d9 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:57:47 +0000 Subject: [PATCH 09/80] #32: Revert pp tests --- lib/ants/tests/fileformats/pp/test_CallbackPP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index a93d964..4fd5b03 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -21,7 +21,7 @@ def setUp(self): def call(self): callback = CallbackPP() callback.append_user_callback(self.user_callback) - callback(self.cube, mock.sentinel.field, "filename") + callback(self.cube, mock.sentinel.field, mock.sentinel.filename) def test__freeze_pseudo_level_is_called_if_pseudo_level_present(self): pseudo_level = iris.coords.DimCoord([0], long_name="pseudo_level") From 3dc860b794ead77f0365772e8be49e9586ef7d06 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:58:11 +0000 Subject: [PATCH 10/80] #32: Remove unused imports --- lib/ants/tests/fileformats/pp/test_CallbackPP.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index 4fd5b03..628299d 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -4,9 +4,6 @@ # See LICENSE.txt in the root of the repository for full licensing details. import unittest.mock as mock -import tempfile -import os -import pytest import ants.tests import iris From da8e951718cffbfc4cc03f8576758e443651ddc3 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Wed, 10 Dec 2025 10:19:40 +0000 Subject: [PATCH 11/80] #32: Commit to save work on adding new load functionality to command line applications --- lib/ants/cli/ancil_2anc.py | 9 +++++---- lib/ants/cli/ancil_fill_n_merge.py | 8 ++++++-- lib/ants/cli/ancil_general_regrid.py | 9 ++++++++- lib/ants/command_parse.py | 12 ++++++++++++ lib/ants/io/load.py | 2 ++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 60deaa1..84e4f54 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,16 @@ import iris -def load_data(source): +def load_data(source, ignore_metadata_files): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source) + cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only): +def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files): """ Convert specified source file to an ancillary. @@ -66,7 +66,7 @@ def main(source_path, output_path, grid_staggering, netcdf_only): written to an ancillary. """ - source_cubes = load_data(source_path) + source_cubes = load_data(source_path, ignore_metadata_files) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering @@ -98,6 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, + args.ignore_metadata_files ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 9bb7cea..41333ab 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -19,6 +19,7 @@ def load_data( primary_source, + ignore_metadata_files, alternate_source=None, validity_polygon_filepath=None, target_mask_filepath=None, @@ -56,12 +57,12 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source) + primary_cubes = ants.io.load.load(primary_source, ignore_metadata_files=ignore_metadata_files) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source) + alternate_cubes = ants.io.load.load(alternate_source,ignore_metadata_files=ignore_metadata_files) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -96,6 +97,7 @@ def main( end, netcdf_only, search_method, + ignore_metadata_files ): """ Perform merge and fill operation on the provided sources. @@ -159,6 +161,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files ) result = primary_cubes @@ -251,6 +254,7 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, + ignore_metadata_files=args.ignore_metadata_files ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 7f6bd86..01fe332 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -37,13 +37,14 @@ def load_data( source, + ignore_metadata_files, target_grid=None, target_landseamask=None, land_fraction_threshold=None, begin=None, end=None, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: @@ -76,6 +77,7 @@ def main( save_ukca, netcdf_only, search_method, + ignore_metadata_files, ): """ General regrid application top level call function. @@ -118,6 +120,9 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. + invert_mask : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns ------- @@ -132,6 +137,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -210,6 +216,7 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, + args.ignore_metadata_files, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 2a3a01c..3a58ed3 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. + --ignore-metadata-files Stops the application from + searching for metadata files + with the same name as the + source and loading them + alongside the source. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) + self.add_argument( + "--ignore-metadata-files", + action="store_true", + help="Turn off the automatic loading of metadata files alongside source " + "files", + required=False, + ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index ee54bc4..155c2f7 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -357,6 +357,8 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False + print("args: ", args) + print("kwargs: ", kwargs) if "ignore_metadata_files" in kwargs: ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( From 79793eed4dd947f9a8ed7095c56a23a8a26bb9cf Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 12:42:58 +0000 Subject: [PATCH 12/80] #32: Working rose stem with new load functionality --- lib/ants/cli/ancil_general_regrid.py | 7 ++++--- lib/ants/io/load.py | 12 ++++++++---- rose-stem/app/ancil_2anc/rose-app.conf | 2 +- rose-stem/app/ancil_create_shapefile/rose-app.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-invert_mask.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-land_cover.conf | 2 +- .../opt/rose-app-grid_to_grid.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- rose-stem/app/ancil_general_regrid/rose-app.conf | 2 +- .../rose-app.conf | 2 +- 10 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 01fe332..94ec6e1 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -37,18 +37,19 @@ def load_data( source, - ignore_metadata_files, target_grid=None, target_landseamask=None, land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid) + target_cube = ants.io.load.load_grid(target_grid, + ignore_metadata_files=ignore_metadata_files) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold @@ -120,7 +121,7 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. - invert_mask : :obj:`bool`, optional + ignore_metadata_files : :obj:`bool`, optional When set to True, files containing metadata will not be loaded alongside data and added as attributes to the cube. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 155c2f7..5dca7d9 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,18 +232,18 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask") + lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") + land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction",ignore_metadata_files=True) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename)[0] + cube = ants.io.load.load(filename, ignore_metadata_files=True)[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -357,9 +357,13 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False + print("the func: ", func) print("args: ", args) print("kwargs: ", kwargs) - if "ignore_metadata_files" in kwargs: + print("boo") + print('ignore_metadata_files' in kwargs) + if 'ignore_metadata_files' in kwargs: + print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( "ignore metadata files: ", diff --git a/rose-stem/app/ancil_2anc/rose-app.conf b/rose-stem/app/ancil_2anc/rose-app.conf index 7ab4ed0..60f9ee4 100644 --- a/rose-stem/app/ancil_2anc/rose-app.conf +++ b/rose-stem/app/ancil_2anc/rose-app.conf @@ -11,7 +11,7 @@ history=${CYLC_WORKFLOW_ID}@${ROSE_SUITE_REVISION}:${ROSE_TASK_NAME} [command] default=ants-launch ancil_2anc.py \ =${source} --grid-staggering ${grid_staggering} -o ${output} \ - =--ants-config ${ANTS_CONFIG} + =--ants-config ${ANTS_CONFIG} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_create_shapefile/rose-app.conf b/rose-stem/app/ancil_create_shapefile/rose-app.conf index 4f37bbf..ac0bbe0 100644 --- a/rose-stem/app/ancil_create_shapefile/rose-app.conf +++ b/rose-stem/app/ancil_create_shapefile/rose-app.conf @@ -1,5 +1,5 @@ [command] -default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} +default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} --ignore-metadata-files [env] OUTPUT=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 5d8e792..88b6cc2 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} + =--search-method ${search_method} --ignore-metadata-files [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index 4db75a3..bd71de6 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore-metadata-files [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf index 23bc985..78e6087 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --search-method ${search_method} + =${begin} ${end} --search-method ${search_method} --ignore-metadata-files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 9a4c4ba..5a7fd0e 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} + =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore-metadata-files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index fb1a949..778e84f 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} + =${begin} ${end} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index 6f166bd..ad44222 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} + =--begin ${begin} --end ${end} --search-method ${search_method} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf From 119dfcf8154e12e4857b6f30592513ff71a80f69 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:16:50 +0000 Subject: [PATCH 13/80] #32: Run black --- lib/ants/cli/ancil_2anc.py | 2 +- lib/ants/cli/ancil_fill_n_merge.py | 14 +++++++++----- lib/ants/cli/ancil_general_regrid.py | 9 ++++++--- lib/ants/io/load.py | 12 ++++++++---- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 84e4f54..48a6f05 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -98,7 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, - args.ignore_metadata_files + args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 41333ab..4f15bd1 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -57,12 +57,16 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source, ignore_metadata_files=ignore_metadata_files) + primary_cubes = ants.io.load.load( + primary_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source,ignore_metadata_files=ignore_metadata_files) + alternate_cubes = ants.io.load.load( + alternate_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -97,7 +101,7 @@ def main( end, netcdf_only, search_method, - ignore_metadata_files + ignore_metadata_files, ): """ Perform merge and fill operation on the provided sources. @@ -161,7 +165,7 @@ def main( land_fraction_threshold, begin, end, - ignore_metadata_files + ignore_metadata_files, ) result = primary_cubes @@ -254,7 +258,7 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, - ignore_metadata_files=args.ignore_metadata_files + ignore_metadata_files=args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 94ec6e1..dd84790 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -44,12 +44,15 @@ def load_data( end=None, ignore_metadata_files=None, ): - source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) + source_cubes = ants.io.load.load( + source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid, - ignore_metadata_files=ignore_metadata_files) + target_cube = ants.io.load.load_grid( + target_grid, ignore_metadata_files=ignore_metadata_files + ) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 5dca7d9..1a69025 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,12 +232,16 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) + lbm = ants.io.load.load_cube( + filename, "land_binary_mask", ignore_metadata_files=True + ) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction",ignore_metadata_files=True) + land_fraction = ants.io.load.load_cube( + filename, "vegetation_area_fraction", ignore_metadata_files=True + ) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: @@ -361,8 +365,8 @@ def load_function(*args, **kwargs): print("args: ", args) print("kwargs: ", kwargs) print("boo") - print('ignore_metadata_files' in kwargs) - if 'ignore_metadata_files' in kwargs: + print("ignore_metadata_files" in kwargs) + if "ignore_metadata_files" in kwargs: print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( From 04b3f9c2359f3c9d4ce2c4149c87b9533a390d7e Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:17:23 +0000 Subject: [PATCH 14/80] #32: Remove unneeded argument flag --- rose-stem/app/ancil_create_shapefile/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/app/ancil_create_shapefile/rose-app.conf b/rose-stem/app/ancil_create_shapefile/rose-app.conf index ac0bbe0..4f37bbf 100644 --- a/rose-stem/app/ancil_create_shapefile/rose-app.conf +++ b/rose-stem/app/ancil_create_shapefile/rose-app.conf @@ -1,5 +1,5 @@ [command] -default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} --ignore-metadata-files +default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} [env] OUTPUT=${ROSE_DATA}/ite.shp From abcdeae83cc29145609835d1739c8618fec20f9e Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 6 Jan 2026 09:38:27 +0000 Subject: [PATCH 15/80] #32: Fix to work with unittests --- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/io/load.py | 2 ++ .../tests/command_parse/test_integration.py | 34 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 4f15bd1..3fee185 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -19,13 +19,13 @@ def load_data( primary_source, - ignore_metadata_files, alternate_source=None, validity_polygon_filepath=None, target_mask_filepath=None, land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None ): """ Load the necessary data for performing a merge and fill operation. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 1a69025..0753226 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -434,6 +434,8 @@ def __call__(self, cube, field, filename): will run the user callback """ print("callback has been added") + if type(filename) is list: + filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 31105ab..cd5f50c 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,6 +30,7 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): + """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -53,6 +54,7 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -69,6 +71,7 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, + ignore_metadata_files=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -150,6 +153,7 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): + """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -176,11 +180,13 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): + """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -207,6 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, + ignore_metadata_files=False ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -265,3 +272,30 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() + + def test_set_ignore_metadata_files_flag(self): + "Tests that ignore_metadata_files will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-files" + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_files=True, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) From 212dd0bb47e3f685fe666f735ee81a332432b037 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 8 Jan 2026 09:40:09 +0000 Subject: [PATCH 16/80] #32: Add working tests --- lib/ants/io/load.py | 29 ++++++++-- .../tests/io/load/test_CallbackMetadata.py | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 0753226..beed200 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -438,6 +438,7 @@ def __call__(self, cube, field, filename): filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) + print("m_file: ", metadata_files) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: @@ -457,15 +458,31 @@ def _retrieve_metadata(self, metadata_files, cube): The cube being loaded. """ + valid_metadata_names = ["license", "attribution", "restrictions"] + other_license= ["lisense", "licence", "lisence"] for metadata_file in metadata_files: - open_file = open(metadata_file, "r") - metadata = open_file.readlines() - open_file.close() file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] - cube.attributes[attribute_name] = metadata - with open("written_license.txt", "a") as file: - file.write("".join(metadata)) + if attribute_name in other_license: + warnings.warn(f"The attribute name {attribute_name} has been changed to " + "license, in line with ANTS working practices.", category=UserWarning) + attribute_name = "license" + if attribute_name not in valid_metadata_names: + warnings.warn(f"Attribute {attribute_name} is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions.", category=UserWarning) + else: + print("cube attributes", cube.attributes) + if attribute_name in cube.attributes: + raise AttributeError(f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag.") + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + cube.attributes[attribute_name] = metadata + with open("written_license.txt", "a") as file: + file.write("".join(metadata)) def load_cube(*args, **kwargs): diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 50a3c5b..71b71dd 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -7,6 +7,7 @@ class directly. import iris import pytest import unittest.mock as mock +import warnings def test_metadata_files_added_to_attributes(tmp_path): @@ -81,3 +82,59 @@ def test_args_parsed_correctly_with_positional_args(tmp_path): iris.save(test_cube, str(temporary_cube_path)) ants.io.load.load_cube(temporary_cube_path, None, mock_callback) mock_callback.assert_called() + +def test_existing_metadata_error(tmp_path): + """Tests that an error is raised when a cube has existing metadata attributes.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes['license'] = 'an existing license' + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + error_message="The license is already an attribute on the cube. To ignore metadata files, use the --ignore-metadata-files flag." + + with pytest.raises(AttributeError, match=error_message): + ants.io.load.load_cube(temporary_cube_path) + +def test_misspelt_license_warning(tmp_path): + """Tests that different spellings for license will raise a warning.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.lisense" + temporary_license_path.write_text(license_text, encoding="utf-8") + + warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) + +def test_misspelt_license_added(tmp_path): + """Tests that a different spelling of license will add a license attribute.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.lisense" + temporary_license_path.write_text(license_text, encoding="utf-8") + # ignore warning that will be raised + warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=warning_message, category=UserWarning + ) + loaded_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_cube.attributes['license'] == ['a license'] + +def test_invalid_metadata_name(tmp_path): + """Tests that an invalid metadata name will not be added as an attribute.""" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" + temporary_license_path.write_text(" ", encoding="utf-8") + + warning_message = "Attribute invalid-name is not a valid metadata file name. Accepted metadata names are license, attribution and restrictions." + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) From ae8aab899e1e69ea5c648b5e253a6afa2b6047b2 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 13 Jan 2026 08:46:54 +0000 Subject: [PATCH 17/80] #32: Run stylechecks --- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/io/load.py | 33 +++++++++------ .../tests/command_parse/test_integration.py | 4 +- .../tests/io/load/test_CallbackMetadata.py | 41 ++++++++++++++----- 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 3fee185..4776891 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,7 +25,7 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, - ignore_metadata_files=None + ignore_metadata_files=None, ): """ Load the necessary data for performing a merge and fill operation. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index beed200..6a4a801 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -56,8 +56,8 @@ """ import copy -import warnings import glob +import warnings from contextlib import contextmanager from functools import wraps @@ -374,9 +374,10 @@ def load_function(*args, **kwargs): ignore_metadata_files, type(ignore_metadata_files), ) - if ignore_metadata_files == False: + if not ignore_metadata_files: print("doing the thing") - # Do the handling for each way a user callback can be passed in through iris + # Do the handling for each way a user callback can be passed in through + # iris user_callback = None if len(args) == 3: user_callback = args[2] @@ -459,24 +460,32 @@ def _retrieve_metadata(self, metadata_files, cube): """ valid_metadata_names = ["license", "attribution", "restrictions"] - other_license= ["lisense", "licence", "lisence"] + other_license = ["lisense", "licence", "lisence"] for metadata_file in metadata_files: file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] if attribute_name in other_license: - warnings.warn(f"The attribute name {attribute_name} has been changed to " - "license, in line with ANTS working practices.", category=UserWarning) + warnings.warn( + f"The attribute name {attribute_name} has been changed to " + "license, in line with ANTS working practices.", + category=UserWarning, + ) attribute_name = "license" if attribute_name not in valid_metadata_names: - warnings.warn(f"Attribute {attribute_name} is not a valid metadata file " - "name. Accepted metadata names are license, attribution " - "and restrictions.", category=UserWarning) + warnings.warn( + f"Attribute {attribute_name} is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions.", + category=UserWarning, + ) else: print("cube attributes", cube.attributes) if attribute_name in cube.attributes: - raise AttributeError(f"The {attribute_name} is already an attribute on the " - "cube. To ignore metadata files, use the " - "--ignore-metadata-files flag.") + raise AttributeError( + f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) open_file = open(metadata_file, "r") metadata = open_file.readlines() open_file.close() diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index cd5f50c..50c11e0 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -213,7 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, - ignore_metadata_files=False + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -282,7 +282,7 @@ def test_set_ignore_metadata_files_flag(self): "/path/to/lsm", "-o", "/path/to/output", - "--ignore-metadata-files" + "--ignore-metadata-files", ] with mock.patch("sys.argv", new=new): parser = AntsArgParser(target_lsm=True) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 71b71dd..5bc3d5d 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -1,13 +1,18 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. """ Includes tests for end-to-end functionality of CallbackMetadata as well as calling the class directly. """ +import unittest.mock as mock +import warnings + import ants.io.load import iris import pytest -import unittest.mock as mock -import warnings def test_metadata_files_added_to_attributes(tmp_path): @@ -83,20 +88,25 @@ def test_args_parsed_correctly_with_positional_args(tmp_path): ants.io.load.load_cube(temporary_cube_path, None, mock_callback) mock_callback.assert_called() + def test_existing_metadata_error(tmp_path): """Tests that an error is raised when a cube has existing metadata attributes.""" license_text = "a license" test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - test_cube.attributes['license'] = 'an existing license' + test_cube.attributes["license"] = "an existing license" temporary_cube_path = tmp_path / "cube_attribute.nc" iris.save(test_cube, str(temporary_cube_path)) temporary_license_path = tmp_path / "cube_attribute.nc.license" temporary_license_path.write_text(license_text, encoding="utf-8") - error_message="The license is already an attribute on the cube. To ignore metadata files, use the --ignore-metadata-files flag." + error_message = ( + "The license is already an attribute on the cube. To ignore " + "metadata files, use the --ignore-metadata-files flag." + ) with pytest.raises(AttributeError, match=error_message): ants.io.load.load_cube(temporary_cube_path) + def test_misspelt_license_warning(tmp_path): """Tests that different spellings for license will raise a warning.""" license_text = "a license" @@ -106,10 +116,14 @@ def test_misspelt_license_warning(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.lisense" temporary_license_path.write_text(license_text, encoding="utf-8") - warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + warning_message = ( + "The attribute name lisense has been changed to license, in " + "line with ANTS working practices." + ) with pytest.raises(UserWarning, match=warning_message): ants.io.load.load_cube(temporary_cube_path) + def test_misspelt_license_added(tmp_path): """Tests that a different spelling of license will add a license attribute.""" license_text = "a license" @@ -119,13 +133,15 @@ def test_misspelt_license_added(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.lisense" temporary_license_path.write_text(license_text, encoding="utf-8") # ignore warning that will be raised - warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + warning_message = ( + "The attribute name lisense has been changed to license, in " + "line with ANTS working practices." + ) with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message=warning_message, category=UserWarning - ) + warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) loaded_cube = ants.io.load.load_cube(temporary_cube_path) - assert loaded_cube.attributes['license'] == ['a license'] + assert loaded_cube.attributes["license"] == ["a license"] + def test_invalid_metadata_name(tmp_path): """Tests that an invalid metadata name will not be added as an attribute.""" @@ -135,6 +151,9 @@ def test_invalid_metadata_name(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" temporary_license_path.write_text(" ", encoding="utf-8") - warning_message = "Attribute invalid-name is not a valid metadata file name. Accepted metadata names are license, attribution and restrictions." + warning_message = ( + "Attribute invalid-name is not a valid metadata file name. " + "Accepted metadata names are license, attribution and restrictions." + ) with pytest.raises(UserWarning, match=warning_message): ants.io.load.load_cube(temporary_cube_path) From 2c00cd47cc1c891aa4ca921e9569712796492bff Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 29 Jan 2026 14:07:14 +0000 Subject: [PATCH 18/80] #32: remove print statements --- lib/ants/io/load.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 6a4a801..b2a4107 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -361,21 +361,9 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False - print("the func: ", func) - print("args: ", args) - print("kwargs: ", kwargs) - print("boo") - print("ignore_metadata_files" in kwargs) if "ignore_metadata_files" in kwargs: - print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") - print( - "ignore metadata files: ", - ignore_metadata_files, - type(ignore_metadata_files), - ) if not ignore_metadata_files: - print("doing the thing") # Do the handling for each way a user callback can be passed in through # iris user_callback = None @@ -384,8 +372,6 @@ def load_function(*args, **kwargs): else: if "callback" in kwargs: user_callback = kwargs.pop("callback") - print("first args: ", args) - print("first kwargs: ", kwargs) args, kwargs = _add_callback( _CallbackMetadata(user_callback), *args, **kwargs ) @@ -434,16 +420,13 @@ def __call__(self, cube, field, filename): The method that runs when iris runs the callback. Collects the filenames and will run the user callback """ - print("callback has been added") if type(filename) is list: filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) - print("m_file: ", metadata_files) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: - print("should have added user callback") self._user_callback(cube, field, filename) def _retrieve_metadata(self, metadata_files, cube): @@ -479,7 +462,6 @@ def _retrieve_metadata(self, metadata_files, cube): category=UserWarning, ) else: - print("cube attributes", cube.attributes) if attribute_name in cube.attributes: raise AttributeError( f"The {attribute_name} is already an attribute on the " From 1519caa1de5de9fa72bdb4e767718bf46e0d06e7 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 29 Jan 2026 16:13:59 +0000 Subject: [PATCH 19/80] add more unit tests --- lib/ants/cli/ancil_2anc.py | 9 +- lib/ants/cli/ancil_fill_n_merge.py | 12 +- lib/ants/cli/ancil_general_regrid.py | 15 +- lib/ants/command_parse.py | 12 -- lib/ants/io/load.py | 39 ++--- .../tests/command_parse/test_integration.py | 34 ---- .../tests/io/load/test_CallbackMetadata.py | 151 ++++++------------ rose-stem/app/ancil_2anc/rose-app.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- .../opt/rose-app-land_cover.conf | 2 +- .../opt/rose-app-grid_to_grid.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- .../app/ancil_general_regrid/rose-app.conf | 2 +- .../rose-app.conf | 2 +- 14 files changed, 74 insertions(+), 212 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 48a6f05..60deaa1 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,16 @@ import iris -def load_data(source, ignore_metadata_files): +def load_data(source): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) + cubes = ants.io.load.load(source) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files): +def main(source_path, output_path, grid_staggering, netcdf_only): """ Convert specified source file to an ancillary. @@ -66,7 +66,7 @@ def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata written to an ancillary. """ - source_cubes = load_data(source_path, ignore_metadata_files) + source_cubes = load_data(source_path) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering @@ -98,7 +98,6 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, - args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 4776891..9bb7cea 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,7 +25,6 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, - ignore_metadata_files=None, ): """ Load the necessary data for performing a merge and fill operation. @@ -57,16 +56,12 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load( - primary_source, ignore_metadata_files=ignore_metadata_files - ) + primary_cubes = ants.io.load.load(primary_source) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load( - alternate_source, ignore_metadata_files=ignore_metadata_files - ) + alternate_cubes = ants.io.load.load(alternate_source) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -101,7 +96,6 @@ def main( end, netcdf_only, search_method, - ignore_metadata_files, ): """ Perform merge and fill operation on the provided sources. @@ -165,7 +159,6 @@ def main( land_fraction_threshold, begin, end, - ignore_metadata_files, ) result = primary_cubes @@ -258,7 +251,6 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, - ignore_metadata_files=args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index dd84790..7f6bd86 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -42,17 +42,12 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, - ignore_metadata_files=None, ): - source_cubes = ants.io.load.load( - source, ignore_metadata_files=ignore_metadata_files - ) + source_cubes = ants.io.load.load(source) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid( - target_grid, ignore_metadata_files=ignore_metadata_files - ) + target_cube = ants.io.load.load_grid(target_grid) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold @@ -81,7 +76,6 @@ def main( save_ukca, netcdf_only, search_method, - ignore_metadata_files, ): """ General regrid application top level call function. @@ -124,9 +118,6 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. - ignore_metadata_files : :obj:`bool`, optional - When set to True, files containing metadata will not be loaded alongside data - and added as attributes to the cube. Returns ------- @@ -141,7 +132,6 @@ def main( land_fraction_threshold, begin, end, - ignore_metadata_files, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -220,7 +210,6 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, - args.ignore_metadata_files, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 3a58ed3..2a3a01c 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,11 +33,6 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. - --ignore-metadata-files Stops the application from - searching for metadata files - with the same name as the - source and loading them - alongside the source. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -176,13 +171,6 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) - self.add_argument( - "--ignore-metadata-files", - action="store_true", - help="Turn off the automatic loading of metadata files alongside source " - "files", - required=False, - ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index b2a4107..9afdef2 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,22 +232,18 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube( - filename, "land_binary_mask", ignore_metadata_files=True - ) + lbm = ants.io.load.load_cube(filename, "land_binary_mask") lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube( - filename, "vegetation_area_fraction", ignore_metadata_files=True - ) + land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename, ignore_metadata_files=True)[0] + cube = ants.io.load.load(filename)[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -360,21 +356,6 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) - ignore_metadata_files = False - if "ignore_metadata_files" in kwargs: - ignore_metadata_files = kwargs.pop("ignore_metadata_files") - if not ignore_metadata_files: - # Do the handling for each way a user callback can be passed in through - # iris - user_callback = None - if len(args) == 3: - user_callback = args[2] - else: - if "callback" in kwargs: - user_callback = kwargs.pop("callback") - args, kwargs = _add_callback( - _CallbackMetadata(user_callback), *args, **kwargs - ) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -454,6 +435,12 @@ def _retrieve_metadata(self, metadata_files, cube): category=UserWarning, ) attribute_name = "license" + if attribute_name in cube.attributes: + raise AttributeError( + f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) if attribute_name not in valid_metadata_names: warnings.warn( f"Attribute {attribute_name} is not a valid metadata file " @@ -462,18 +449,10 @@ def _retrieve_metadata(self, metadata_files, cube): category=UserWarning, ) else: - if attribute_name in cube.attributes: - raise AttributeError( - f"The {attribute_name} is already an attribute on the " - "cube. To ignore metadata files, use the " - "--ignore-metadata-files flag." - ) open_file = open(metadata_file, "r") metadata = open_file.readlines() open_file.close() cube.attributes[attribute_name] = metadata - with open("written_license.txt", "a") as file: - file.write("".join(metadata)) def load_cube(*args, **kwargs): diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 50c11e0..31105ab 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,7 +30,6 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): - """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -54,7 +53,6 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, - ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -71,7 +69,6 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, - ignore_metadata_files=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -153,7 +150,6 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): - """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -180,13 +176,11 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, - ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): - """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -213,7 +207,6 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, - ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -272,30 +265,3 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() - - def test_set_ignore_metadata_files_flag(self): - "Tests that ignore_metadata_files will be set to True when passed in." - new = [ - "program", - "/path/to/source", - "--target-lsm", - "/path/to/lsm", - "-o", - "/path/to/output", - "--ignore-metadata-files", - ] - with mock.patch("sys.argv", new=new): - parser = AntsArgParser(target_lsm=True) - args = parser.parse_args() - - target_args = argparse.Namespace( - ants_config=None, - land_threshold=None, - target_lsm="/path/to/lsm", - output="/path/to/output", - sources=["/path/to/source"], - netcdf_only=False, - ignore_metadata_files=True, - ) - self.assertFalse(self.mock_config.called) - self.assertEqual(args, target_args) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 5bc3d5d..a081613 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -15,48 +15,6 @@ class directly. import pytest -def test_metadata_files_added_to_attributes(tmp_path): - """Tests that metadata files are found and added to the cube's attributes.""" - # The text that would be in a license file - license_text = """This is the license of the cube. - - It should be preserved and added to the cube when loaded. - """ - # How the text should look while stored in an array - loaded_license = [ - "This is the license of the cube.\n", - "\n", - " It should be preserved and added to the cube when loaded.\n", - " ", - ] - test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - temporary_cube_path = tmp_path / "cube_attribute.pp" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.pp.license" - temporary_license_path.write_text(license_text, encoding="utf-8") - loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) - assert loaded_test_cube.attributes["license"] == loaded_license - - -def test_no_metadata_loaded(tmp_path): - """Tests that metadata files are not loaded when the option is turned off.""" - # The text that would be in a license file - license_text = """This is the license of the cube. - - It should be preserved and added to the cube when loaded. - """ - test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - temporary_cube_path = tmp_path / "cube_attribute.pp" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.pp.license" - temporary_license_path.write_text(license_text, encoding="utf-8") - loaded_test_cube = ants.io.load.load_cube( - temporary_cube_path, ignore_metadata_files=True - ) - with pytest.raises(KeyError): - loaded_test_cube.attributes["license"] - - def test_user_callback_added(): """Test that on ititialisation, the user's function will be set.""" @@ -68,7 +26,7 @@ def user_callback(cube, field, filename): def test_args_parsed_correctly_with_kwargs(tmp_path): - """Tests that when passed a callback using a keyword argument, + """Tests that when passed a user callback using a keyword argument, the callback is parsed correctly.""" mock_callback = mock.Mock() test_cube = ants.tests.stock.geodetic(shape=(2, 2)) @@ -79,7 +37,7 @@ def test_args_parsed_correctly_with_kwargs(tmp_path): def test_args_parsed_correctly_with_positional_args(tmp_path): - """Tests that when passed a callback using a keyword argument, + """Tests that when passed a user callback using a keyword argument, the callback is parsed correctly.""" mock_callback = mock.Mock() test_cube = ants.tests.stock.geodetic(shape=(2, 2)) @@ -89,71 +47,62 @@ def test_args_parsed_correctly_with_positional_args(tmp_path): mock_callback.assert_called() -def test_existing_metadata_error(tmp_path): - """Tests that an error is raised when a cube has existing metadata attributes.""" - license_text = "a license" +def test__retrieve_metadata_correct_name(): + """Tests that correctly named metadata files are going to be read.""" + class_instance = ants.io.load._CallbackMetadata(None) + mock_file_open = mock.mock_open() test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - test_cube.attributes["license"] = "an existing license" - temporary_cube_path = tmp_path / "cube_attribute.nc" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.nc.license" - temporary_license_path.write_text(license_text, encoding="utf-8") - error_message = ( - "The license is already an attribute on the cube. To ignore " - "metadata files, use the --ignore-metadata-files flag." - ) + path = ["fake-path/fake-cube.attribution"] + with mock.patch("builtins.open", mock_file_open): + class_instance._retrieve_metadata(path, test_cube) + mock_file_open.assert_called_once_with("fake-path/fake-cube.attribution", "r") - with pytest.raises(AttributeError, match=error_message): - ants.io.load.load_cube(temporary_cube_path) - -def test_misspelt_license_warning(tmp_path): - """Tests that different spellings for license will raise a warning.""" - license_text = "a license" +def test__retrieve_metadata_incorrect_name(): + """Tests that files without a valid metadata name won't be read.""" + class_instance = ants.io.load._CallbackMetadata(None) test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - temporary_cube_path = tmp_path / "cube_attribute.nc" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.nc.lisense" - temporary_license_path.write_text(license_text, encoding="utf-8") - - warning_message = ( - "The attribute name lisense has been changed to license, in " - "line with ANTS working practices." + path = ["fake-path/fake-cube.pp"] + expected_message = ( + "Attribute pp is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions." ) - with pytest.raises(UserWarning, match=warning_message): - ants.io.load.load_cube(temporary_cube_path) + with pytest.raises(UserWarning, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) -def test_misspelt_license_added(tmp_path): - """Tests that a different spelling of license will add a license attribute.""" - license_text = "a license" +def test_attribute_already_on_cube(): + """Tests that having both metadata attributes and sidecar files of the same type + will raise an attribute error.""" + class_instance = ants.io.load._CallbackMetadata(None) test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - temporary_cube_path = tmp_path / "cube_attribute.nc" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.nc.lisense" - temporary_license_path.write_text(license_text, encoding="utf-8") - # ignore warning that will be raised - warning_message = ( - "The attribute name lisense has been changed to license, in " - "line with ANTS working practices." + test_cube.attributes["attribution"] = "This is a attribution. " + path = ["fake-path/fake-cube.attribution"] + expected_message = ( + "The attribution is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." ) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) - loaded_cube = ants.io.load.load_cube(temporary_cube_path) - assert loaded_cube.attributes["license"] == ["a license"] - - -def test_invalid_metadata_name(tmp_path): - """Tests that an invalid metadata name will not be added as an attribute.""" + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +@pytest.mark.filterwarnings( + "ignore:The attribute name lisense has been changed to license, in line with ANTS " + "working practices.:UserWarning" +) +def test_missplet_license_with_licensed_cube(): + """Tests that when given a sidecar file with a miss-spelt license and a cube with an + existing license, an error will be given.""" + class_instance = ants.io.load._CallbackMetadata(None) test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - temporary_cube_path = tmp_path / "cube_attribute.nc" - iris.save(test_cube, str(temporary_cube_path)) - temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" - temporary_license_path.write_text(" ", encoding="utf-8") - - warning_message = ( - "Attribute invalid-name is not a valid metadata file name. " - "Accepted metadata names are license, attribution and restrictions." + test_cube.attributes["license"] = "This is a license. " + path = ["fake-path/fake-cube.lisense"] + expected_message = ( + "The license is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." ) - with pytest.raises(UserWarning, match=warning_message): - ants.io.load.load_cube(temporary_cube_path) + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) diff --git a/rose-stem/app/ancil_2anc/rose-app.conf b/rose-stem/app/ancil_2anc/rose-app.conf index 60f9ee4..7ab4ed0 100644 --- a/rose-stem/app/ancil_2anc/rose-app.conf +++ b/rose-stem/app/ancil_2anc/rose-app.conf @@ -11,7 +11,7 @@ history=${CYLC_WORKFLOW_ID}@${ROSE_SUITE_REVISION}:${ROSE_TASK_NAME} [command] default=ants-launch ancil_2anc.py \ =${source} --grid-staggering ${grid_staggering} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --ignore-metadata-files + =--ants-config ${ANTS_CONFIG} [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 88b6cc2..5d8e792 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} --ignore-metadata-files + =--search-method ${search_method} [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index bd71de6..4db75a3 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore-metadata-files + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf index 78e6087..23bc985 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --search-method ${search_method} --ignore-metadata-files + =${begin} ${end} --search-method ${search_method} [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 5a7fd0e..9a4c4ba 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore-metadata-files + =--invert-mask ${begin} ${end} --search-method ${search_method} [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index 778e84f..fb1a949 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --ignore-metadata-files + =${begin} ${end} [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index ad44222..6f166bd 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} --ignore-metadata-files + =--begin ${begin} --end ${end} --search-method ${search_method} [env] ANTS_CONFIG=rose-app-run.conf From 0af2eceb4b5ebcc60be08220f161ae3d98a1b295 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 29 Jan 2026 16:23:45 +0000 Subject: [PATCH 20/80] remove unused import --- lib/ants/tests/io/load/test_CallbackMetadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index a081613..ebe2b07 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -8,7 +8,6 @@ class directly. """ import unittest.mock as mock -import warnings import ants.io.load import iris From 8f49436a3a6c2317ec3687aae62cd61f9e1824b0 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 31 Mar 2026 12:57:25 +0100 Subject: [PATCH 21/80] Update iris punctuation Co-authored-by: Andrew Clark <2562650+arjclark@users.noreply.github.com> Signed-off-by: Theo Geddes --- lib/ants/io/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9afdef2..92053ae 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -398,7 +398,7 @@ def __init__(self, user_callback): def __call__(self, cube, field, filename): """ - The method that runs when iris runs the callback. Collects the filenames and + The method that runs when Iris runs the callback. Collects the filenames and will run the user callback """ if type(filename) is list: From 95538de8339d6adf3701b22a2427a50e0c0794c6 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 2 Apr 2026 10:33:15 +0100 Subject: [PATCH 22/80] Update docstring in _add_callback() --- lib/ants/io/load.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 92053ae..8cff074 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -375,6 +375,12 @@ def _add_callback(callback, *args, **kwargs): """ Adds both the ants callback and the user provided callback (if any) to the load. + + Parameters + ---------- + callback: :class:`_CallbackMetadata` + An object that will contain the ants call back and an attribute + with the user callback if applicable. """ args = list(args) if len(args) == 1: From 8b3334992dd9cb49b80d5bc51b0646c76100659c Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 2 Apr 2026 13:14:31 +0100 Subject: [PATCH 23/80] Add speechmarks around license --- lib/ants/io/load.py | 2 +- lib/ants/tests/io/load/test_CallbackMetadata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 8cff074..69aab25 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -437,7 +437,7 @@ def _retrieve_metadata(self, metadata_files, cube): if attribute_name in other_license: warnings.warn( f"The attribute name {attribute_name} has been changed to " - "license, in line with ANTS working practices.", + "'license', in line with ANTS working practices.", category=UserWarning, ) attribute_name = "license" diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index ebe2b07..6b0450b 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -88,7 +88,7 @@ def test_attribute_already_on_cube(): @pytest.mark.filterwarnings( - "ignore:The attribute name lisense has been changed to license, in line with ANTS " + "ignore:The attribute name lisense has been changed to 'license', in line with ANTS " "working practices.:UserWarning" ) def test_missplet_license_with_licensed_cube(): From 077b0dba0365f729f16060bc30dd9c05b8295596 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 2 Apr 2026 13:30:21 +0100 Subject: [PATCH 24/80] Update _CallbackMetadata docstring --- lib/ants/io/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 69aab25..9ae2ccb 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -395,7 +395,7 @@ class _CallbackMetadata(object): """Callback for collecting metadata from sidecar files. This callback will load additional metadata files with the naming convention: - filename. and append the contents of those files to the cube + filename.[license,attribution,restrictions] and append the contents of those files to the cube attributes. """ From 4bbb16640edb6a5bfd1634fc08f1fe6717c38bd5 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 7 Apr 2026 16:16:15 +0100 Subject: [PATCH 25/80] Remove checks for misspelling --- lib/ants/io/load.py | 4 ++-- lib/ants/tests/io/load/test_CallbackMetadata.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9ae2ccb..af42cb6 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -430,11 +430,11 @@ def _retrieve_metadata(self, metadata_files, cube): """ valid_metadata_names = ["license", "attribution", "restrictions"] - other_license = ["lisense", "licence", "lisence"] + other_license = "licence" for metadata_file in metadata_files: file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] - if attribute_name in other_license: + if attribute_name == other_license: warnings.warn( f"The attribute name {attribute_name} has been changed to " "'license', in line with ANTS working practices.", diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 6b0450b..c502b7e 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -88,7 +88,7 @@ def test_attribute_already_on_cube(): @pytest.mark.filterwarnings( - "ignore:The attribute name lisense has been changed to 'license', in line with ANTS " + "ignore:The attribute name licence has been changed to 'license', in line with ANTS " "working practices.:UserWarning" ) def test_missplet_license_with_licensed_cube(): @@ -97,7 +97,7 @@ def test_missplet_license_with_licensed_cube(): class_instance = ants.io.load._CallbackMetadata(None) test_cube = ants.tests.stock.geodetic(shape=(2, 2)) test_cube.attributes["license"] = "This is a license. " - path = ["fake-path/fake-cube.lisense"] + path = ["fake-path/fake-cube.licence"] expected_message = ( "The license is already an attribute on the " "cube. To ignore metadata files, use the " From 1761aeec0ad4dabfc72cba003fbd8ca614217a99 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 8 Apr 2026 10:28:52 +0100 Subject: [PATCH 26/80] Changes implementing the loading of metadata --- lib/ants/io/load.py | 21 +++++- .../tests/io/load/test_CallbackMetadata.py | 75 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index af42cb6..8455618 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,18 +232,18 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask") + lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") + land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction", ignore_metadata_files=True) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename)[0] + cube = ants.io.load.load(filename, ignore_metadata_files=True)[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -356,6 +356,21 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) + ignore_metadata_files = False + if "ignore_metadata_files" in kwargs: + ignore_metadata_files = kwargs.pop("ignore_metadata_files") + if not ignore_metadata_files: + # Do the handling for each way a user callback can be passed in through + # iris + user_callback = None + if len(args) == 3: + user_callback = args[2] + else: + if "callback" in kwargs: + user_callback = kwargs.pop("callback") + args, kwargs = _add_callback( + _CallbackMetadata(user_callback), *args, **kwargs + ) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index c502b7e..50a8db6 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -8,12 +8,54 @@ class directly. """ import unittest.mock as mock +import warnings import ants.io.load import iris import pytest +def test_metadata_files_added_to_attributes(tmp_path): + """Tests that metadata files are found and added to the cube's attributes.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + # How the text should look while stored in an array + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_test_cube.attributes["license"] == loaded_license + + +def test_no_metadata_loaded(tmp_path): + """Tests that metadata files are not loaded when the option is turned off.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube( + temporary_cube_path, ignore_metadata_files=True + ) + with pytest.raises(KeyError): + loaded_test_cube.attributes["license"] + def test_user_callback_added(): """Test that on ititialisation, the user's function will be set.""" @@ -105,3 +147,36 @@ def test_missplet_license_with_licensed_cube(): ) with pytest.raises(AttributeError, match=expected_message): class_instance._retrieve_metadata(path, test_cube) + +def test_misspelt_license_added(tmp_path): + """Tests that a different spelling of license will add a license attribute.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.licence" + temporary_license_path.write_text(license_text, encoding="utf-8") + # ignore warning that will be raised + warning_message = ( + "The attribute name licence has been changed to 'license', in " + "line with ANTS working practices." + ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) + loaded_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_cube.attributes["license"] == ["a license"] + +def test_invalid_metadata_name(tmp_path): + """Tests that an invalid metadata name will not be added as an attribute.""" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" + temporary_license_path.write_text(" ", encoding="utf-8") + + warning_message = ( + "Attribute invalid-name is not a valid metadata file name. " + "Accepted metadata names are license, attribution and restrictions." + ) + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) From 23f56efb73faa76d631bcf1a7092028a32945518 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 8 Apr 2026 10:33:11 +0100 Subject: [PATCH 27/80] Run black --- lib/ants/io/load.py | 8 ++++++-- lib/ants/tests/io/load/test_CallbackMetadata.py | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 8455618..9806d6f 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,12 +232,16 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) + lbm = ants.io.load.load_cube( + filename, "land_binary_mask", ignore_metadata_files=True + ) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction", ignore_metadata_files=True) + land_fraction = ants.io.load.load_cube( + filename, "vegetation_area_fraction", ignore_metadata_files=True + ) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 50a8db6..d9f9bf7 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -56,6 +56,7 @@ def test_no_metadata_loaded(tmp_path): with pytest.raises(KeyError): loaded_test_cube.attributes["license"] + def test_user_callback_added(): """Test that on ititialisation, the user's function will be set.""" @@ -148,6 +149,7 @@ def test_missplet_license_with_licensed_cube(): with pytest.raises(AttributeError, match=expected_message): class_instance._retrieve_metadata(path, test_cube) + def test_misspelt_license_added(tmp_path): """Tests that a different spelling of license will add a license attribute.""" license_text = "a license" @@ -166,6 +168,7 @@ def test_misspelt_license_added(tmp_path): loaded_cube = ants.io.load.load_cube(temporary_cube_path) assert loaded_cube.attributes["license"] == ["a license"] + def test_invalid_metadata_name(tmp_path): """Tests that an invalid metadata name will not be added as an attribute.""" test_cube = ants.tests.stock.geodetic(shape=(2, 2)) From 03bf80011de55da364b5ceb70f3c8df7ae835d42 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 8 Apr 2026 14:31:38 +0100 Subject: [PATCH 28/80] fix flake8 line lengths --- lib/ants/io/load.py | 4 ++-- lib/ants/tests/io/load/test_CallbackMetadata.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9806d6f..1c393fc 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -414,8 +414,8 @@ class _CallbackMetadata(object): """Callback for collecting metadata from sidecar files. This callback will load additional metadata files with the naming convention: - filename.[license,attribution,restrictions] and append the contents of those files to the cube - attributes. + filename.[license,attribution,restrictions] and append the contents of those files + to the cube attributes. """ def __init__(self, user_callback): diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index d9f9bf7..892b8ac 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -131,8 +131,8 @@ def test_attribute_already_on_cube(): @pytest.mark.filterwarnings( - "ignore:The attribute name licence has been changed to 'license', in line with ANTS " - "working practices.:UserWarning" + "ignore:The attribute name licence has been changed to 'license', in line with " + "ANTS working practices.:UserWarning" ) def test_missplet_license_with_licensed_cube(): """Tests that when given a sidecar file with a miss-spelt license and a cube with an From 106714d06f739178880bc1b64a947d3e0526ab32 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 22 Apr 2026 14:34:10 +0100 Subject: [PATCH 29/80] Add command line support --- lib/ants/cli/ancil_2anc.py | 9 ++--- lib/ants/cli/ancil_fill_n_merge.py | 19 ++++++++-- lib/ants/cli/ancil_general_regrid.py | 15 ++++++-- lib/ants/command_parse.py | 12 +++++++ .../tests/command_parse/test_integration.py | 35 +++++++++++++++++++ rose-stem/app/ancil_2anc/rose-app.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- .../opt/rose-app-land_cover.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- .../app/ancil_general_regrid/rose-app.conf | 2 +- .../rose-app.conf | 2 +- 11 files changed, 88 insertions(+), 14 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 60deaa1..84e4f54 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,16 @@ import iris -def load_data(source): +def load_data(source, ignore_metadata_files): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source) + cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only): +def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files): """ Convert specified source file to an ancillary. @@ -66,7 +66,7 @@ def main(source_path, output_path, grid_staggering, netcdf_only): written to an ancillary. """ - source_cubes = load_data(source_path) + source_cubes = load_data(source_path, ignore_metadata_files) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering @@ -98,6 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, + args.ignore_metadata_files ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 9bb7cea..a9252ef 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,6 +25,7 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): """ Load the necessary data for performing a merge and fill operation. @@ -45,6 +46,9 @@ def load_data( Datetime to start the processing. end: :obj:`datetime`, optional Datetime to end the processing. + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns @@ -56,12 +60,16 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source) + primary_cubes = ants.io.load.load( + primary_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source) + alternate_cubes = ants.io.load.load( + alternate_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -96,6 +104,7 @@ def main( end, netcdf_only, search_method, + ignore_metadata_files, ): """ Perform merge and fill operation on the provided sources. @@ -141,6 +150,10 @@ def main( search_method : :obj:`str` Select the search method to be used when filling missing points. The methods currently supported are "spiral" and "kdtree". + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. + Returns ------- : :class:`~iris.cube.CubeList` @@ -159,6 +172,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files ) result = primary_cubes @@ -251,6 +265,7 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, + ignore_metadata_files=args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 7f6bd86..8589e8e 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -42,12 +42,17 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load( + source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid) + target_cube = ants.io.load.load_grid( + target_grid, ignore_metadata_files=ignore_metadata_files + ) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold @@ -76,6 +81,7 @@ def main( save_ukca, netcdf_only, search_method, + ignore_metadata_files, ): """ General regrid application top level call function. @@ -118,6 +124,9 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns ------- @@ -132,6 +141,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -210,6 +220,7 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, + args.ignore_metadata_files, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 2a3a01c..e40cb0c 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. + --ignore-metadata-files Stops the application from + searching for metadata files + with the same name as the + source and loading them + alongside the source. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) + self.add_argument( + "--ignore-metadata-files", + action="store_true", + help="Stops the application from searching for metadata files with " + "the same name as the source and loading them alongside the source.", + required=False, + ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 31105ab..fe4fc40 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,6 +30,7 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): + """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -53,6 +54,7 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -69,6 +71,7 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, + ignore_metadata_files=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -150,6 +153,7 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): + """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -176,11 +180,13 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): + """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -207,6 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -265,3 +272,31 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() + + def test_set_ignore_metadata_files_flag(self): + "Tests that ignore_metadata_files will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-files", + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_files=True, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) + diff --git a/rose-stem/app/ancil_2anc/rose-app.conf b/rose-stem/app/ancil_2anc/rose-app.conf index 7ab4ed0..60f9ee4 100644 --- a/rose-stem/app/ancil_2anc/rose-app.conf +++ b/rose-stem/app/ancil_2anc/rose-app.conf @@ -11,7 +11,7 @@ history=${CYLC_WORKFLOW_ID}@${ROSE_SUITE_REVISION}:${ROSE_TASK_NAME} [command] default=ants-launch ancil_2anc.py \ =${source} --grid-staggering ${grid_staggering} -o ${output} \ - =--ants-config ${ANTS_CONFIG} + =--ants-config ${ANTS_CONFIG} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 5d8e792..1bc2cec 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} + =--search-method ${search_method} --ignore_metadata_files [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index 4db75a3..4d63e55 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore_metadata_files [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 9a4c4ba..30166ce 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} + =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore_metadata_files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index fb1a949..b3e7520 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} + =${begin} ${end} --ignore_metadata_files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index 6f166bd..d6771f6 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} + =--begin ${begin} --end ${end} --search-method ${search_method} --ignore_metadata_files [env] ANTS_CONFIG=rose-app-run.conf From 1835b2691c895d859059d34a7853f4cca67a558e Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 22 Apr 2026 16:39:37 +0100 Subject: [PATCH 30/80] Update extra argument to be correct --- rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf | 2 +- rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf | 2 +- .../app/ancil_general_regrid/opt/rose-app-invert_mask.conf | 2 +- rose-stem/app/ancil_general_regrid/rose-app.conf | 2 +- .../app/ancil_general_regrid_with_time_constraint/rose-app.conf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 1bc2cec..88b6cc2 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} --ignore_metadata_files + =--search-method ${search_method} --ignore-metadata-files [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index 4d63e55..bd71de6 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore_metadata_files + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore-metadata-files [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 30166ce..5a7fd0e 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore_metadata_files + =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore-metadata-files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index b3e7520..778e84f 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --ignore_metadata_files + =${begin} ${end} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index d6771f6..ad44222 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} --ignore_metadata_files + =--begin ${begin} --end ${end} --search-method ${search_method} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf From 3d37136a17cd4d7c30215acec1905af6c05536da Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 22 Apr 2026 16:53:13 +0100 Subject: [PATCH 31/80] Add full command parse integration and test --- lib/ants/cli/ancil_2anc.py | 2 +- lib/ants/cli/ancil_fill_n_merge.py | 6 +++--- lib/ants/cli/ancil_general_regrid.py | 4 ++-- lib/ants/tests/command_parse/test_integration.py | 1 - 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 84e4f54..48a6f05 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -98,7 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, - args.ignore_metadata_files + args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index a9252ef..96dc0a7 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -62,14 +62,14 @@ def load_data( """ primary_cubes = ants.io.load.load( primary_source, ignore_metadata_files=ignore_metadata_files - ) + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: alternate_cubes = ants.io.load.load( alternate_source, ignore_metadata_files=ignore_metadata_files - ) + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -172,7 +172,7 @@ def main( land_fraction_threshold, begin, end, - ignore_metadata_files + ignore_metadata_files, ) result = primary_cubes diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 8589e8e..dd84790 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -46,13 +46,13 @@ def load_data( ): source_cubes = ants.io.load.load( source, ignore_metadata_files=ignore_metadata_files - ) + ) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: target_cube = ants.io.load.load_grid( target_grid, ignore_metadata_files=ignore_metadata_files - ) + ) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index fe4fc40..50c11e0 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -299,4 +299,3 @@ def test_set_ignore_metadata_files_flag(self): ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) - From 8b5447e1be462edde9c8be0879af2fc76a12d7fc Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 8 Jun 2026 14:22:48 +0100 Subject: [PATCH 32/80] partial commit to save work on adding rose stem test --- rose-stem/app/ancil_2anc/rose-app.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-invert_mask.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-land_cover.conf | 2 +- .../opt/rose-app-metadata_file.conf | 3 +++ .../opt/rose-app-invert_mask.conf | 2 +- rose-stem/app/ancil_general_regrid/rose-app.conf | 2 +- .../rose-app.conf | 2 +- .../app/rose_ana/opt/rose-app-fill_n_merge.conf | 1 + rose-stem/flow.cylc | 12 +++++++++--- 9 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf diff --git a/rose-stem/app/ancil_2anc/rose-app.conf b/rose-stem/app/ancil_2anc/rose-app.conf index 60f9ee4..7ab4ed0 100644 --- a/rose-stem/app/ancil_2anc/rose-app.conf +++ b/rose-stem/app/ancil_2anc/rose-app.conf @@ -11,7 +11,7 @@ history=${CYLC_WORKFLOW_ID}@${ROSE_SUITE_REVISION}:${ROSE_TASK_NAME} [command] default=ants-launch ancil_2anc.py \ =${source} --grid-staggering ${grid_staggering} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --ignore-metadata-files + =--ants-config ${ANTS_CONFIG} [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 88b6cc2..5d8e792 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} --ignore-metadata-files + =--search-method ${search_method} [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index bd71de6..4db75a3 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore-metadata-files + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf new file mode 100644 index 0000000..c6bc106 --- /dev/null +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf @@ -0,0 +1,3 @@ +[env] + +source=/data/users/theo.geddes/ants-source/cci_ukv.nc \ No newline at end of file diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 5a7fd0e..9a4c4ba 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore-metadata-files + =--invert-mask ${begin} ${end} --search-method ${search_method} [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index 778e84f..fb1a949 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --ignore-metadata-files + =${begin} ${end} [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index ad44222..6f166bd 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} --ignore-metadata-files + =--begin ${begin} --end ${end} --search-method ${search_method} [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf b/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf index 4315909..2063854 100644 --- a/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf +++ b/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf @@ -11,3 +11,4 @@ filelist=ancil_fill_n_merge_land_cover_spiral =ancil_fill_n_merge_invert_mask_kdtree.nc =ancil_fill_n_merge_land_cover_latitude_weighted_kdtree.nc =ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree.nc + =ancil_fill_n_merge_metadata_file.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 48941eb..9acd870 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -2,7 +2,7 @@ {% set ANTS_MODULE = "ants/developer" %} {% set PYTHONPATH_PREPEND = "$CYLC_WORKFLOW_RUN_DIR/share/fcm_make_ants/build/lib" %} -{% set fill_n_merge_source = ['land_cover', 'invert_mask'] %} +{% set fill_n_merge_source = ['land_cover', 'invert_mask', 'metadata_file'] %} {% set grid_source = ['grid_to_grid', 'grid_to_variable_resolution_grid', 'grid_to_n48e_namelist', 'grid_to_n48_namelist', '3d_to_3d', '3d_to_3d_with_extrapolation', 'invert_mask'] %} {% set target_lsm_grid_sources = ['grid_to_grid', 'invert_mask'] %} @@ -13,7 +13,7 @@ {%- set name_graphs = { "install_graph" : "install_cold", "ancil_2anc_graph" : "install_cold => ancil_2anc => rose_ana_2anc:fail? => plot_comparisons_2anc", - "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", + "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_metadata_file & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "fill_n_merge_invert_mask_graph" : "install_cold => ancil_fill_n_merge_invert_mask & ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "general_regrid_grid_to_grid_graph" : "install_cold => ancil_general_regrid_grid_to_grid => rose_ana_general_regrid:fail? => plot_comparisons_general_regrid install_cold => ancil_general_regrid_grid_to_grid_latitude_weighted_kdtree => rose_ana_general_regrid_latitude_weighted_kdtree:fail? => plot_comparisons_general_regrid_latitude_weighted_kdtree", @@ -185,7 +185,7 @@ else module unload {{ ANTS_MODULE }} fi ''' - +ancil_fill_n_merge_metadata_file & {% for source in fill_n_merge_source %} [[ANCIL_FILL_N_MERGE_{{ source }}]] inherit = ANTS_CORE, LARGE @@ -197,8 +197,14 @@ fi [[ancil_fill_n_merge_{{ source }}_latitude_weighted_kdtree]] inherit = ANCIL_FILL_N_MERGE_{{ source }} script = rose task-run --app-key=ancil_fill_n_merge -O {{ source }} -O latitude-weighted-kdtree + + [[ancil_fill_n_merge_metadata_file]] + inherit = ANCIL_FILL_N_MERGE_{{ source }} + script = rose task-run --app-key=ancil_fill_n_merge -O {{ source }} {% endfor %} + + [[ancil_create_ite_shapefile]] inherit = ANTS_CORE, SMALL [[[environment]]] From b808ac98b99bbde10f31ecff025989f154bb992b Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 11 Jun 2026 12:54:54 +0100 Subject: [PATCH 33/80] update rose stem --- .../opt/rose-app-metadata_file.conf | 11 +++++++++-- .../opt/rose-app-metadata.conf | 15 +++++++++++++++ .../app/rose_ana/opt/rose-app-general_regrid.conf | 2 ++ rose-stem/flow.cylc | 12 ++++++++++-- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf index c6bc106..c3e608c 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf @@ -1,3 +1,10 @@ -[env] +[command] +default=ants-launch ancil_fill_n_merge.py \ + =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} -source=/data/users/theo.geddes/ants-source/cci_ukv.nc \ No newline at end of file +[env] +polygon=${ROSE_DATA}/ite.shp +search_method=kdtree +source=${TEST_SOURCES_DIR}/ancil_fill_n_merge/ancil_lct_ite.nc /data/users/theo.geddes/ants-source/cci_ukv.nc +target=${TEST_SOURCES_DIR}/ancil_fill_n_merge/ukv_coarse_land_cover_fraction.nc diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf new file mode 100644 index 0000000..f3429ea --- /dev/null +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf @@ -0,0 +1,15 @@ +[ants_decomposition] +x_split=0 +y_split=0 + +[command] +default=ants-launch ancil_general_regrid.py \ + =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ + =${begin} ${end} --search-method ${search_method} + +[env] +output=${ROSE_DATA}/${ROSE_TASK_NAME} +search_method=kdtree +source=${TEST_SOURCES_DIR}/ancil_general_regrid/n96e_orca_land_cover_fraction +target=${TEST_SOURCES_DIR}/ancil_general_regrid/qrparm.mask.nc +target_type=lsm diff --git a/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf b/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf index 487f9de..6f7127b 100644 --- a/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf +++ b/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf @@ -29,3 +29,5 @@ filelist=ancil_general_regrid_grid_to_variable_resolution_grid_split1 =ancil_general_regrid_grid_to_n48e_namelist_split0.nc =ancil_general_regrid_3d_to_3d_split0.nc =ancil_general_regrid_3d_to_3d_with_extrapolation_split0.nc + =ancil_general_regrid_metadata + =ancil_general_regrid_metadata.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 9acd870..cc9ec68 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -13,7 +13,9 @@ {%- set name_graphs = { "install_graph" : "install_cold", "ancil_2anc_graph" : "install_cold => ancil_2anc => rose_ana_2anc:fail? => plot_comparisons_2anc", - "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_metadata_file & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", + "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", + "load_metadata_graph" : "install_cold => ancil_fill_n_merge_metadata_file => rose_ana_fill_n_merge", + "load_metadata_regrid_graph" : "install_cold => ancil_general_regrid_metadata => rose_ana_general_regrid", "fill_n_merge_invert_mask_graph" : "install_cold => ancil_fill_n_merge_invert_mask & ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "general_regrid_grid_to_grid_graph" : "install_cold => ancil_general_regrid_grid_to_grid => rose_ana_general_regrid:fail? => plot_comparisons_general_regrid install_cold => ancil_general_regrid_grid_to_grid_latitude_weighted_kdtree => rose_ana_general_regrid_latitude_weighted_kdtree:fail? => plot_comparisons_general_regrid_latitude_weighted_kdtree", @@ -185,7 +187,7 @@ else module unload {{ ANTS_MODULE }} fi ''' -ancil_fill_n_merge_metadata_file & + {% for source in fill_n_merge_source %} [[ANCIL_FILL_N_MERGE_{{ source }}]] inherit = ANTS_CORE, LARGE @@ -215,6 +217,7 @@ ancil_fill_n_merge_metadata_file & [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = fill_n_merge + ANTS_KGO_DIRECTORY_OVERRIDE = /home/users/theo.geddes/cylc-run/dev-ants-core/run78/share/data [[ANCIL_2ANC]] inherit=ANTS_CORE, LARGE @@ -260,11 +263,16 @@ ancil_fill_n_merge_metadata_file & inherit=ANCIL_GENERAL_REGRID_WITH_TIME_CONSTRAINT script = rose task-run --app-key=ancil_general_regrid_with_time_constraint -O latitude-weighted-kdtree + [[ancil_general_regrid_metadata]] + inherit = ANTS_CORE, ANCIL_GENERAL_REGRID, LARGE + script = rose task-run --app-key=ancil_general_regrid -O metadata + [[rose_ana_general_regrid]] inherit = ROSE_ANA [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = general_regrid + ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run83/share/data/ [[rose_ana_general_regrid]] inherit = ROSE_ANA From d9c6d056ea3beefb5c57b3a243c32e4492c29a40 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 11 Jun 2026 14:21:28 +0100 Subject: [PATCH 34/80] Clean up rose stem to only have general regrid additions --- .../ancil_fill_n_merge/opt/rose-app-metadata_file.conf | 10 ---------- rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf | 1 - rose-stem/flow.cylc | 8 +------- 3 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf deleted file mode 100644 index c3e608c..0000000 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-metadata_file.conf +++ /dev/null @@ -1,10 +0,0 @@ -[command] -default=ants-launch ancil_fill_n_merge.py \ - =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} - -[env] -polygon=${ROSE_DATA}/ite.shp -search_method=kdtree -source=${TEST_SOURCES_DIR}/ancil_fill_n_merge/ancil_lct_ite.nc /data/users/theo.geddes/ants-source/cci_ukv.nc -target=${TEST_SOURCES_DIR}/ancil_fill_n_merge/ukv_coarse_land_cover_fraction.nc diff --git a/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf b/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf index 2063854..4315909 100644 --- a/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf +++ b/rose-stem/app/rose_ana/opt/rose-app-fill_n_merge.conf @@ -11,4 +11,3 @@ filelist=ancil_fill_n_merge_land_cover_spiral =ancil_fill_n_merge_invert_mask_kdtree.nc =ancil_fill_n_merge_land_cover_latitude_weighted_kdtree.nc =ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree.nc - =ancil_fill_n_merge_metadata_file.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index cc9ec68..127689f 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -2,7 +2,7 @@ {% set ANTS_MODULE = "ants/developer" %} {% set PYTHONPATH_PREPEND = "$CYLC_WORKFLOW_RUN_DIR/share/fcm_make_ants/build/lib" %} -{% set fill_n_merge_source = ['land_cover', 'invert_mask', 'metadata_file'] %} +{% set fill_n_merge_source = ['land_cover', 'invert_mask'] %} {% set grid_source = ['grid_to_grid', 'grid_to_variable_resolution_grid', 'grid_to_n48e_namelist', 'grid_to_n48_namelist', '3d_to_3d', '3d_to_3d_with_extrapolation', 'invert_mask'] %} {% set target_lsm_grid_sources = ['grid_to_grid', 'invert_mask'] %} @@ -14,7 +14,6 @@ "install_graph" : "install_cold", "ancil_2anc_graph" : "install_cold => ancil_2anc => rose_ana_2anc:fail? => plot_comparisons_2anc", "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", - "load_metadata_graph" : "install_cold => ancil_fill_n_merge_metadata_file => rose_ana_fill_n_merge", "load_metadata_regrid_graph" : "install_cold => ancil_general_regrid_metadata => rose_ana_general_regrid", "fill_n_merge_invert_mask_graph" : "install_cold => ancil_fill_n_merge_invert_mask & ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "general_regrid_grid_to_grid_graph" : "install_cold => ancil_general_regrid_grid_to_grid => rose_ana_general_regrid:fail? => plot_comparisons_general_regrid @@ -199,10 +198,6 @@ fi [[ancil_fill_n_merge_{{ source }}_latitude_weighted_kdtree]] inherit = ANCIL_FILL_N_MERGE_{{ source }} script = rose task-run --app-key=ancil_fill_n_merge -O {{ source }} -O latitude-weighted-kdtree - - [[ancil_fill_n_merge_metadata_file]] - inherit = ANCIL_FILL_N_MERGE_{{ source }} - script = rose task-run --app-key=ancil_fill_n_merge -O {{ source }} {% endfor %} @@ -217,7 +212,6 @@ fi [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = fill_n_merge - ANTS_KGO_DIRECTORY_OVERRIDE = /home/users/theo.geddes/cylc-run/dev-ants-core/run78/share/data [[ANCIL_2ANC]] inherit=ANTS_CORE, LARGE From d71456b6e71696f30351fc0e4067a6bd6f8311a2 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 24 Jun 2026 09:05:59 +0100 Subject: [PATCH 35/80] remove whitespcae --- rose-stem/flow.cylc | 2 -- 1 file changed, 2 deletions(-) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 127689f..21ecd3a 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -200,8 +200,6 @@ fi script = rose task-run --app-key=ancil_fill_n_merge -O {{ source }} -O latitude-weighted-kdtree {% endfor %} - - [[ancil_create_ite_shapefile]] inherit = ANTS_CORE, SMALL [[[environment]]] From 4ec679b4bc711b4151198f2aed01eb904927676d Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 24 Jun 2026 09:38:49 +0100 Subject: [PATCH 36/80] add functionality for checking for multiple attributes to be saved --- lib/ants/io/save.py | 16 +++++++++ .../save/test__check_multiple_attributes.py | 36 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 lib/ants/tests/io/save/test__check_multiple_attributes.py diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 9cbc83e..04cb182 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -322,3 +322,19 @@ def _update_history_cmd(cube): items[0] = os.path.basename(items[0]) items.append("({})".format(metadata)) if metadata else None ants.utils.cube.update_history(cc, " ".join(items), date) + + +def _check_multiple_attributes(attribute_list, cube_names): + """Checks whether the attribute can be written out exactly as is, or if it has to be + pre-pended with the cube name.""" + # check if multiple things in list + if len(attribute_list) == 1: + return attribute_list + # check if attributes are the same + if len(set(attribute_list)) == 1: + return attribute_list[:1] + # if they are not the same, add the cube name + concatonated_attribute = [] + for attribute, name in zip(attribute_list, cube_names, strict=True): + concatonated_attribute.append(name + " = " + attribute) + return concatonated_attribute diff --git a/lib/ants/tests/io/save/test__check_multiple_attributes.py b/lib/ants/tests/io/save/test__check_multiple_attributes.py new file mode 100644 index 0000000..5b19a9c --- /dev/null +++ b/lib/ants/tests/io/save/test__check_multiple_attributes.py @@ -0,0 +1,36 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +from ants.io.save import _check_multiple_attributes + + +def test_one_element_in_list(): + """Tests that when given a list with one element, that is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + +def test_all_elements_in_list_same(): + """Tests that when all elements in list are the same, one is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + attribute_list = [ + "This is the only attribution given. ", + "This is the only attribution given. ", + "This is the only attribution given. ", + ] + actual = _check_multiple_attributes(attribute_list, cube_name) + assert expected == actual + + +def test_different_elements_have_cube_names(): + """Tests that when multiple attributes are given, cube names are included.""" + expected = ["cube1 = license 1. ", "cube2 = license 2. "] + licenses = ["license 1. ", "license 2. "] + cube_names = ["cube1", "cube2"] + actual = _check_multiple_attributes(licenses, cube_names) + assert expected == actual From f79cc0d66558368db2b04d7d0b7b5f452f58b0d4 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 24 Jun 2026 10:55:30 +0100 Subject: [PATCH 37/80] Fix spelling Co-authored-by: Andrew Clark <2562650+arjclark@users.noreply.github.com> Signed-off-by: Theo Geddes --- lib/ants/io/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 04cb182..5f54ea6 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -334,7 +334,7 @@ def _check_multiple_attributes(attribute_list, cube_names): if len(set(attribute_list)) == 1: return attribute_list[:1] # if they are not the same, add the cube name - concatonated_attribute = [] + concatenated_attribute = [] for attribute, name in zip(attribute_list, cube_names, strict=True): concatonated_attribute.append(name + " = " + attribute) return concatonated_attribute From 9b71d8fcb7fb6cd6ef6d7f5b612fe77f2b370ab8 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 24 Jun 2026 10:56:44 +0100 Subject: [PATCH 38/80] Fix variable spelling Co-authored-by: Andrew Clark <2562650+arjclark@users.noreply.github.com> Signed-off-by: Theo Geddes --- lib/ants/io/save.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 5f54ea6..73e9680 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -336,5 +336,5 @@ def _check_multiple_attributes(attribute_list, cube_names): # if they are not the same, add the cube name concatenated_attribute = [] for attribute, name in zip(attribute_list, cube_names, strict=True): - concatonated_attribute.append(name + " = " + attribute) - return concatonated_attribute + concatenated_attribute.append(name + " = " + attribute) + return concatenated_attribute From 7fcbbc309cb58a5345f5d49cf3d9ecdd5b26ab54 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 24 Jun 2026 11:20:30 +0100 Subject: [PATCH 39/80] Add handling for one single cube being passed in --- lib/ants/io/save.py | 3 +++ .../tests/io/save/test__check_multiple_attributes.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 73e9680..3dfeaac 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -333,6 +333,9 @@ def _check_multiple_attributes(attribute_list, cube_names): # check if attributes are the same if len(set(attribute_list)) == 1: return attribute_list[:1] + # check if there is only one cube + if len(cube_names) == 1: + return attribute_list # if they are not the same, add the cube name concatenated_attribute = [] for attribute, name in zip(attribute_list, cube_names, strict=True): diff --git a/lib/ants/tests/io/save/test__check_multiple_attributes.py b/lib/ants/tests/io/save/test__check_multiple_attributes.py index 5b19a9c..ae534ed 100644 --- a/lib/ants/tests/io/save/test__check_multiple_attributes.py +++ b/lib/ants/tests/io/save/test__check_multiple_attributes.py @@ -14,6 +14,17 @@ def test_one_element_in_list(): assert expected == actual +def test_only_one_Cube_in_list(): + """Tests that when given only one cube name, the attributes are returned.""" + expected = [ + "This is an attribution for some of the data. ", + "This is a different attribution for the rest of the data.", + ] + cube_name = ["a cube with multiple attributions"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + def test_all_elements_in_list_same(): """Tests that when all elements in list are the same, one is returned.""" expected = ["This is the only attribution given. "] From 9e7e27a8a819e2f4e4549badbc3ba0819376324b Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 10:41:34 +0100 Subject: [PATCH 40/80] add the ability to write metadata to sidecar files --- lib/ants/cli/ancil_2anc.py | 4 +- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/cli/ancil_general_regrid.py | 6 +- lib/ants/io/save.py | 65 ++++++++- .../tests/io/save/test__write_metadata.py | 123 ++++++++++++++++++ .../opt/rose-app-metadata.conf | 2 +- rose-stem/flow.cylc | 2 +- 7 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 lib/ants/tests/io/save/test__write_metadata.py diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 48a6f05..39f46b3 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -72,7 +72,9 @@ def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata source_cube.attributes["grid_staggering"] = grid_staggering if not netcdf_only: - save.ancil(source_cubes, output_path) + save.ancil( + source_cubes, output_path, ignore_external_metadata=ignore_metadata_files + ) save.netcdf(source_cubes, output_path) return source_cubes diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 96dc0a7..0db80c8 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -182,7 +182,7 @@ def main( ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method) if not netcdf_only: - save.ancil(result, output) + save.ancil(result, output, ignore_external_metadata=ignore_metadata_files) save.netcdf(result, output) return result diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index dd84790..bc0b88c 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -159,7 +159,11 @@ def main( save.ukca_netcdf(regridded_cubes, output_path) else: if not netcdf_only: - save.ancil(regridded_cubes, output_path) + save.ancil( + regridded_cubes, + output_path, + ignore_external_metadata=ignore_metadata_files, + ) save.netcdf(regridded_cubes, output_path) return regridded_cubes diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 3dfeaac..5ccf458 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -27,6 +27,7 @@ import ants.utils.cube import iris +import numpy as np from ants.fileformats.ancil import _cubes_to_ancilfile, _mule_set_lbuser2 from ants.fileformats.netcdf.cf import ( _coerce_netcdf_classic_dtypes, @@ -35,7 +36,7 @@ from ants.fileformats.netcdf.ukca import LOCAL_ATTS, _ukca_conventions -def ancil(cubes, filename): +def ancil(cubes, filename, ignore_external_metadata=False): """ Save one or more cubes to a F03 UM ancillary file. @@ -73,6 +74,9 @@ def ancil(cubes, filename): One or more cubes to be saved. filename : str The name of the F03 UM ancillary file, including any extension. + ignore_external_metadata : bool + Determines whether attributes should be saved to a seperate metadata file. + Default behavior is false, so will write out the metadata. Notes ----- @@ -92,6 +96,8 @@ def ancil(cubes, filename): raise ValueError("F03 UM ancillary files cannot be saved with a .nc extension.") cubes = ants.utils.cube.as_cubelist(cubes) + if not ignore_external_metadata: + _write_metadata(cubes, filename) ancilfile = _cubes_to_ancilfile(cubes) _mule_set_lbuser2(ancilfile) ancilfile.to_file(filename) @@ -324,6 +330,47 @@ def _update_history_cmd(cube): ants.utils.cube.update_history(cc, " ".join(items), date) +def _write_metadata(cubes, filename): + """Check for metadata in the cubes and write out external files + Parameters + ---------- + cubes : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList` + One or more cubes to be saved. + filename : str + The name of the file where the data will be saved to. + """ + license = [] + license_names = [] + attribution = [] + attribution_names = [] + restrictions = [] + restrictions_names = [] + for cube in cubes: + for key, value in cube.attributes.items(): + if key == "license": + license.append(value) + license_names.append(cube.name()) + if key == "attribution": + attribution.append(value) + attribution_names.append(cube.name()) + if key == "restrictions": + restrictions.append(value) + restrictions_names.append(cube.name()) + if len(license) > 0: + writable_license = _check_multiple_attributes(license, license_names) + _write_metadata_file(writable_license, filename, "license") + if len(attribution) > 0: + writable_attribution = _check_multiple_attributes( + attribution, attribution_names + ) + _write_metadata_file(writable_attribution, filename, "attribution") + if len(restrictions) > 0: + writable_restrictions = _check_multiple_attributes( + restrictions, restrictions_names + ) + _write_metadata_file(writable_restrictions, filename, "restrictions") + + def _check_multiple_attributes(attribute_list, cube_names): """Checks whether the attribute can be written out exactly as is, or if it has to be pre-pended with the cube name.""" @@ -341,3 +388,19 @@ def _check_multiple_attributes(attribute_list, cube_names): for attribute, name in zip(attribute_list, cube_names, strict=True): concatenated_attribute.append(name + " = " + attribute) return concatenated_attribute + + +def _write_metadata_file(metadata, filename, attribute_name): + """Takes a list of metadata and writes it to a file called filename.attribute_name. + If for any reason, the file to be written already exists, the new metadata will be + appended to it. + """ + filepath = str(filename) + "." + attribute_name + # flatten list, if metadata contains list of list - possible in cases where metadata + # is being read in + if any(isinstance(element, list) for element in metadata): + metadata = np.concatenate(metadata).tolist() + with open(filepath, "a") as metadata_file: + metadata_file.writelines(metadata) + print("saved") + warnings.warn(f"{attribute_name} has been written to sidecar file {filepath}") diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__write_metadata.py new file mode 100644 index 0000000..1c19e52 --- /dev/null +++ b/lib/ants/tests/io/save/test__write_metadata.py @@ -0,0 +1,123 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +import os +from unittest import mock + +import ants.tests.stock as stock +import pytest +from ants.io.save import _write_metadata + + +@pytest.mark.filterwarnings( + "ignore:license has been written to sidecar file /var/tmp/:UserWarning" +) +def test_license_attribute_written(tmp_path): + """Tests that a cube with a license is written out.""" + cube = stock.geodetic(shape=(2, 2)) + license = "This is a cube's license. " + cube.attributes["license"] = license + cube.rename("license test cube") + filename = tmp_path / "test_cube" + _write_metadata([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == license + + +@pytest.mark.filterwarnings( + "ignore:license has been written to sidecar file /var/tmp/:UserWarning" +) +def test_loaded_license_written(tmp_path): + """Tests that a cube with a license that resembles the format of a longer license + written in, is written out correctly.""" + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + cube = stock.geodetic(shape=(2, 2)) + cube.attributes["license"] = loaded_license + cube.rename("loaded license test cube") + filename = tmp_path / "test_cube" + _write_metadata([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.readlines() + assert actual_license == loaded_license + + +def test_warning_given(): + """Tests that a warning is given when file is written out.""" + cube = stock.geodetic(shape=(2, 2)) + attribution = "This date came from an institution. " + cube.attributes["attribution"] = attribution + expected_message = ( + "attribution has been written to sidecar file filename.attribution" + ) + # mocks out the opening of files, so no file is created + with mock.patch("builtins.open"): + with pytest.raises(UserWarning, match=expected_message): + _write_metadata([cube], "filename") + + +@pytest.mark.filterwarnings( + "ignore:license has been written to sidecar file /var/tmp/:UserWarning" +) +def test_multiple_cubes(tmp_path): + """Test that multiple_cubes with attributes is written out correctly.""" + # creating three cubes with different license attributes + cube1 = stock.geodetic(shape=(2, 2)) + license1 = "This is a cube's license. " + cube1.attributes["license"] = license1 + cube1.rename("the first cube") + cube2 = stock.geodetic(shape=(2, 2)) + license2 = "This is another cube's license. " + cube2.attributes["license"] = license2 + cube2.rename("the second cube") + cube3 = stock.geodetic(shape=(2, 2)) + license3 = "This is a third cube's license. " + cube3.attributes["license"] = license3 + cube3.rename("the third cube") + cubelist = [cube1, cube2, cube3] + filename = tmp_path / "multiple_cube_test" + # The actual test + _write_metadata(cubelist, filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + expected_license = ( + "the first cube = This is a cube's license. the second cube = " + "This is another cube's license. the third cube = This is a third cube's " + "license. " + ) + assert actual_license == expected_license + + +@pytest.mark.filterwarnings( + "ignore:license has been written to sidecar file /var/tmp/:UserWarning" +) +@pytest.mark.filterwarnings( + "ignore:attribution has been written to sidecar file /var/tmp/:UserWarning" +) +@pytest.mark.filterwarnings( + "ignore:restrictions has been written to sidecar file /var/tmp/:UserWarning" +) +def test_all_different_attributes_written_out(tmp_path): + """Tests that a cube with a multiple different attributes writes + out all metadata files.""" + cube = stock.geodetic(shape=(2, 2)) + attribution = "This date came from an institution. " + cube.attributes["attribution"] = attribution + cube.attributes["restrictions"] = ( + "This data is restricted to be used for testing purposes only." + ) + cube.attributes["license"] = "This is a license for the data" + filename = tmp_path / "test_multiple_attributes" + _write_metadata([cube], filename) + assert os.path.exists(str(filename) + ".attribution") + assert os.path.exists(str(filename) + ".restrictions") + assert os.path.exists(str(filename) + ".license") diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf index f3429ea..9b79396 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf @@ -10,6 +10,6 @@ default=ants-launch ancil_general_regrid.py \ [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} search_method=kdtree -source=${TEST_SOURCES_DIR}/ancil_general_regrid/n96e_orca_land_cover_fraction +source=/data/users/theo.geddes/ants-source/n96e_orca_land_cover_fraction target=${TEST_SOURCES_DIR}/ancil_general_regrid/qrparm.mask.nc target_type=lsm diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 21ecd3a..499411d 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -264,7 +264,7 @@ fi [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = general_regrid - ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run83/share/data/ + ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run95/share/data/ [[rose_ana_general_regrid]] inherit = ROSE_ANA From a0e13e8c5cb241089ed56c316d5eeeb131a16639 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 10:56:33 +0100 Subject: [PATCH 41/80] Update rose stem source for roseana --- rose-stem/flow.cylc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 499411d..bb3cad4 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -264,7 +264,7 @@ fi [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = general_regrid - ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run95/share/data/ + ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run98/share/data/ [[rose_ana_general_regrid]] inherit = ROSE_ANA From e1b0595324bc7050e69c7e2038caf90fac54a384 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 13:18:57 +0100 Subject: [PATCH 42/80] Apply suggestions from code review Co-authored-by: Andrew Clark <2562650+arjclark@users.noreply.github.com> Signed-off-by: Theo Geddes --- lib/ants/io/save.py | 4 ++-- lib/ants/tests/io/save/test__write_metadata.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 5ccf458..2c4da99 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -76,7 +76,7 @@ def ancil(cubes, filename, ignore_external_metadata=False): The name of the F03 UM ancillary file, including any extension. ignore_external_metadata : bool Determines whether attributes should be saved to a seperate metadata file. - Default behavior is false, so will write out the metadata. + Default setting is False, so will write out the metadata. Notes ----- @@ -397,7 +397,7 @@ def _write_metadata_file(metadata, filename, attribute_name): """ filepath = str(filename) + "." + attribute_name # flatten list, if metadata contains list of list - possible in cases where metadata - # is being read in + # is being read in if any(isinstance(element, list) for element in metadata): metadata = np.concatenate(metadata).tolist() with open(filepath, "a") as metadata_file: diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__write_metadata.py index 1c19e52..81a976f 100644 --- a/lib/ants/tests/io/save/test__write_metadata.py +++ b/lib/ants/tests/io/save/test__write_metadata.py @@ -53,7 +53,7 @@ def test_loaded_license_written(tmp_path): def test_warning_given(): """Tests that a warning is given when file is written out.""" cube = stock.geodetic(shape=(2, 2)) - attribution = "This date came from an institution. " + attribution = "This data came from an institution. " cube.attributes["attribution"] = attribution expected_message = ( "attribution has been written to sidecar file filename.attribution" @@ -110,7 +110,7 @@ def test_all_different_attributes_written_out(tmp_path): """Tests that a cube with a multiple different attributes writes out all metadata files.""" cube = stock.geodetic(shape=(2, 2)) - attribution = "This date came from an institution. " + attribution = "This data came from an institution. " cube.attributes["attribution"] = attribution cube.attributes["restrictions"] = ( "This data is restricted to be used for testing purposes only." From 5e70363cbd7c9380d5a6ac7c3a92578fa22c7ce1 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 13:40:41 +0100 Subject: [PATCH 43/80] Change name of variable for turning off the save option --- lib/ants/cli/ancil_2anc.py | 4 +++- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/cli/ancil_general_regrid.py | 2 +- lib/ants/io/save.py | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 39f46b3..2d24b97 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -73,7 +73,9 @@ def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata if not netcdf_only: save.ancil( - source_cubes, output_path, ignore_external_metadata=ignore_metadata_files + source_cubes, + output_path, + ignore_writing_metadata_files=ignore_metadata_files, ) save.netcdf(source_cubes, output_path) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 0db80c8..caa29ca 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -182,7 +182,7 @@ def main( ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method) if not netcdf_only: - save.ancil(result, output, ignore_external_metadata=ignore_metadata_files) + save.ancil(result, output, ignore_writing_metadata_files=ignore_metadata_files) save.netcdf(result, output) return result diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index bc0b88c..355226a 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -162,7 +162,7 @@ def main( save.ancil( regridded_cubes, output_path, - ignore_external_metadata=ignore_metadata_files, + ignore_writing_metadata_files=ignore_metadata_files, ) save.netcdf(regridded_cubes, output_path) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 2c4da99..f64bca0 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -36,7 +36,7 @@ from ants.fileformats.netcdf.ukca import LOCAL_ATTS, _ukca_conventions -def ancil(cubes, filename, ignore_external_metadata=False): +def ancil(cubes, filename, ignore_writing_metadata_files=False): """ Save one or more cubes to a F03 UM ancillary file. @@ -74,7 +74,7 @@ def ancil(cubes, filename, ignore_external_metadata=False): One or more cubes to be saved. filename : str The name of the F03 UM ancillary file, including any extension. - ignore_external_metadata : bool + ignore_writing_metadata_files : bool Determines whether attributes should be saved to a seperate metadata file. Default setting is False, so will write out the metadata. @@ -96,7 +96,7 @@ def ancil(cubes, filename, ignore_external_metadata=False): raise ValueError("F03 UM ancillary files cannot be saved with a .nc extension.") cubes = ants.utils.cube.as_cubelist(cubes) - if not ignore_external_metadata: + if not ignore_writing_metadata_files: _write_metadata(cubes, filename) ancilfile = _cubes_to_ancilfile(cubes) _mule_set_lbuser2(ancilfile) From 84808571ecf5becccdfce7284cde222a963a8242 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 13:53:59 +0100 Subject: [PATCH 44/80] Clarify variable name --- lib/ants/io/save.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index f64bca0..f3f1fc2 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -330,13 +330,13 @@ def _update_history_cmd(cube): ants.utils.cube.update_history(cc, " ".join(items), date) -def _write_metadata(cubes, filename): +def _write_metadata(cubes, data_filepath): """Check for metadata in the cubes and write out external files Parameters ---------- cubes : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList` One or more cubes to be saved. - filename : str + data_filepath : str The name of the file where the data will be saved to. """ license = [] @@ -358,17 +358,17 @@ def _write_metadata(cubes, filename): restrictions_names.append(cube.name()) if len(license) > 0: writable_license = _check_multiple_attributes(license, license_names) - _write_metadata_file(writable_license, filename, "license") + _write_metadata_file(writable_license, data_filepath, "license") if len(attribution) > 0: writable_attribution = _check_multiple_attributes( attribution, attribution_names ) - _write_metadata_file(writable_attribution, filename, "attribution") + _write_metadata_file(writable_attribution, data_filepath, "attribution") if len(restrictions) > 0: writable_restrictions = _check_multiple_attributes( restrictions, restrictions_names ) - _write_metadata_file(writable_restrictions, filename, "restrictions") + _write_metadata_file(writable_restrictions, data_filepath, "restrictions") def _check_multiple_attributes(attribute_list, cube_names): From acbf778f935dfba015af2d0375d097e568c88238 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 14:22:01 +0100 Subject: [PATCH 45/80] Remove print statement --- lib/ants/io/save.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index f3f1fc2..b3db582 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -402,5 +402,4 @@ def _write_metadata_file(metadata, filename, attribute_name): metadata = np.concatenate(metadata).tolist() with open(filepath, "a") as metadata_file: metadata_file.writelines(metadata) - print("saved") warnings.warn(f"{attribute_name} has been written to sidecar file {filepath}") From f1d7906b6c183ca8d93f9efab86474f1fd84e5a1 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 14:54:48 +0100 Subject: [PATCH 46/80] Update name and docstring of function --- lib/ants/io/save.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index b3db582..5498960 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -97,7 +97,7 @@ def ancil(cubes, filename, ignore_writing_metadata_files=False): cubes = ants.utils.cube.as_cubelist(cubes) if not ignore_writing_metadata_files: - _write_metadata(cubes, filename) + _check_and_sort_metadata_attributes(cubes, filename) ancilfile = _cubes_to_ancilfile(cubes) _mule_set_lbuser2(ancilfile) ancilfile.to_file(filename) @@ -330,8 +330,9 @@ def _update_history_cmd(cube): ants.utils.cube.update_history(cc, " ".join(items), date) -def _write_metadata(cubes, data_filepath): - """Check for metadata in the cubes and write out external files +def _check_and_sort_metadata_attributes(cubes, data_filepath): + """Checks for a license, attribution or restrictions in the metadata of the cubes, + and calls `_write_metadata_file` for each attribute. Parameters ---------- cubes : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList` From 3ef2e612f46d7b4e391964906e12b80e9f5621f8 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 29 Jun 2026 15:23:21 +0100 Subject: [PATCH 47/80] change docstring syntax --- lib/ants/io/save.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 5498960..7ce683e 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -392,7 +392,8 @@ def _check_multiple_attributes(attribute_list, cube_names): def _write_metadata_file(metadata, filename, attribute_name): - """Takes a list of metadata and writes it to a file called filename.attribute_name. + """Takes a list of metadata and writes it to a file called + filename.. If for any reason, the file to be written already exists, the new metadata will be appended to it. """ From 7c5d69c641149f26b95718a4632db5398f8b59b3 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 09:01:55 +0100 Subject: [PATCH 48/80] Update unittest to use mock --- .../tests/io/save/test__write_metadata.py | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__write_metadata.py index 81a976f..7ca3e28 100644 --- a/lib/ants/tests/io/save/test__write_metadata.py +++ b/lib/ants/tests/io/save/test__write_metadata.py @@ -2,12 +2,11 @@ # # This file is part of ANTS and is released under the BSD 3-Clause license. # See LICENSE.txt in the root of the repository for full licensing details. -import os from unittest import mock import ants.tests.stock as stock import pytest -from ants.io.save import _write_metadata +from ants.io.save import _check_and_sort_metadata_attributes @pytest.mark.filterwarnings( @@ -20,7 +19,7 @@ def test_license_attribute_written(tmp_path): cube.attributes["license"] = license cube.rename("license test cube") filename = tmp_path / "test_cube" - _write_metadata([cube], filename) + _check_and_sort_metadata_attributes([cube], filename) expected_filename = str(filename) + ".license" with open(expected_filename, "r") as file: actual_license = file.read() @@ -43,7 +42,7 @@ def test_loaded_license_written(tmp_path): cube.attributes["license"] = loaded_license cube.rename("loaded license test cube") filename = tmp_path / "test_cube" - _write_metadata([cube], filename) + _check_and_sort_metadata_attributes([cube], filename) expected_filename = str(filename) + ".license" with open(expected_filename, "r") as file: actual_license = file.readlines() @@ -61,7 +60,7 @@ def test_warning_given(): # mocks out the opening of files, so no file is created with mock.patch("builtins.open"): with pytest.raises(UserWarning, match=expected_message): - _write_metadata([cube], "filename") + _check_and_sort_metadata_attributes([cube], "filename") @pytest.mark.filterwarnings( @@ -85,7 +84,7 @@ def test_multiple_cubes(tmp_path): cubelist = [cube1, cube2, cube3] filename = tmp_path / "multiple_cube_test" # The actual test - _write_metadata(cubelist, filename) + _check_and_sort_metadata_attributes(cubelist, filename) expected_filename = str(filename) + ".license" with open(expected_filename, "r") as file: actual_license = file.read() @@ -97,16 +96,7 @@ def test_multiple_cubes(tmp_path): assert actual_license == expected_license -@pytest.mark.filterwarnings( - "ignore:license has been written to sidecar file /var/tmp/:UserWarning" -) -@pytest.mark.filterwarnings( - "ignore:attribution has been written to sidecar file /var/tmp/:UserWarning" -) -@pytest.mark.filterwarnings( - "ignore:restrictions has been written to sidecar file /var/tmp/:UserWarning" -) -def test_all_different_attributes_written_out(tmp_path): +def test_all_different_attributes_written_out(): """Tests that a cube with a multiple different attributes writes out all metadata files.""" cube = stock.geodetic(shape=(2, 2)) @@ -116,8 +106,24 @@ def test_all_different_attributes_written_out(tmp_path): "This data is restricted to be used for testing purposes only." ) cube.attributes["license"] = "This is a license for the data" - filename = tmp_path / "test_multiple_attributes" - _write_metadata([cube], filename) - assert os.path.exists(str(filename) + ".attribution") - assert os.path.exists(str(filename) + ".restrictions") - assert os.path.exists(str(filename) + ".license") + filename = "test_multiple_attributes" + with mock.patch("ants.io.save._write_metadata_file") as mock_method: + _check_and_sort_metadata_attributes([cube], filename) + + expected_license = mock.call( + ["This is a license for the data"], "test_multiple_attributes", "license" + ) + expected_attribution = mock.call( + ["This data came from an institution. "], + "test_multiple_attributes", + "attribution", + ) + expected_restrictions = mock.call( + ["This data is restricted to be used for testing purposes only."], + "test_multiple_attributes", + "restrictions", + ) + + assert expected_license in mock_method.call_args_list + assert expected_attribution in mock_method.call_args_list + assert expected_restrictions in mock_method.call_args_list From 7453293be9d9322dc6381f59565d4499527bb28a Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 09:40:42 +0100 Subject: [PATCH 49/80] Update warning to log info --- lib/ants/io/save.py | 4 +++- lib/ants/tests/io/save/test__write_metadata.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 7ce683e..fa633f6 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -20,6 +20,7 @@ specifying ``saver='ukca'`` (see :func:`ants.io.save.ukca_netcdf`). """ +import logging import os import sys import warnings @@ -404,4 +405,5 @@ def _write_metadata_file(metadata, filename, attribute_name): metadata = np.concatenate(metadata).tolist() with open(filepath, "a") as metadata_file: metadata_file.writelines(metadata) - warnings.warn(f"{attribute_name} has been written to sidecar file {filepath}") + _LOGGER = logging.getLogger(__name__) + _LOGGER.info(f"{attribute_name} has been written to sidecar file {filepath}") diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__write_metadata.py index 7ca3e28..79325d7 100644 --- a/lib/ants/tests/io/save/test__write_metadata.py +++ b/lib/ants/tests/io/save/test__write_metadata.py @@ -2,6 +2,7 @@ # # This file is part of ANTS and is released under the BSD 3-Clause license. # See LICENSE.txt in the root of the repository for full licensing details. +import logging from unittest import mock import ants.tests.stock as stock @@ -49,8 +50,8 @@ def test_loaded_license_written(tmp_path): assert actual_license == loaded_license -def test_warning_given(): - """Tests that a warning is given when file is written out.""" +def test_log_output(caplog): + """Tests that the logger gives the correct output when a file is written out.""" cube = stock.geodetic(shape=(2, 2)) attribution = "This data came from an institution. " cube.attributes["attribution"] = attribution @@ -59,8 +60,9 @@ def test_warning_given(): ) # mocks out the opening of files, so no file is created with mock.patch("builtins.open"): - with pytest.raises(UserWarning, match=expected_message): + with caplog.at_level(logging.INFO): _check_and_sort_metadata_attributes([cube], "filename") + assert expected_message in caplog.text @pytest.mark.filterwarnings( From 67301b059de72f4a2e91f10d4ff75e1ae4e07c05 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 09:53:59 +0100 Subject: [PATCH 50/80] Update code comment to be more clear --- lib/ants/io/save.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index fa633f6..a47395f 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -399,8 +399,8 @@ def _write_metadata_file(metadata, filename, attribute_name): appended to it. """ filepath = str(filename) + "." + attribute_name - # flatten list, if metadata contains list of list - possible in cases where metadata - # is being read in + # Order metadata to be in one list, if metadata contains list of list - possible in + # cases where metadata is being read in if any(isinstance(element, list) for element in metadata): metadata = np.concatenate(metadata).tolist() with open(filepath, "a") as metadata_file: From cc97f9f6c0b01a2457ca1a59df137007147061d7 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 11:44:39 +0100 Subject: [PATCH 51/80] Rewrite check_and_sort_metadata to use a dictionary --- lib/ants/io/save.py | 47 +++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index a47395f..0e80db4 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -341,36 +341,29 @@ def _check_and_sort_metadata_attributes(cubes, data_filepath): data_filepath : str The name of the file where the data will be saved to. """ - license = [] - license_names = [] - attribution = [] - attribution_names = [] - restrictions = [] - restrictions_names = [] + # a dictionary to contain all of the metadata to be saved + metadata_dictionary = {} + # a dictionary to keep track of which cubes have metadata + cube_names_dictionary = {} + # a list of approved attributes that can be saved + attributes_to_save = ["license", "attribution", "restrictions"] for cube in cubes: for key, value in cube.attributes.items(): - if key == "license": - license.append(value) - license_names.append(cube.name()) - if key == "attribution": - attribution.append(value) - attribution_names.append(cube.name()) - if key == "restrictions": - restrictions.append(value) - restrictions_names.append(cube.name()) - if len(license) > 0: - writable_license = _check_multiple_attributes(license, license_names) - _write_metadata_file(writable_license, data_filepath, "license") - if len(attribution) > 0: - writable_attribution = _check_multiple_attributes( - attribution, attribution_names + # check the attribute is one we want to save + if key in attributes_to_save: + if key in metadata_dictionary: + metadata_dictionary[key].append(value) + cube_names_dictionary[key + "_names"].append(cube.name()) + else: + metadata_dictionary[key] = [value] + cube_names_dictionary[key + "_names"] = [cube.name()] + for key, value in metadata_dictionary.items(): + # sort the metadata ready to save + metadata_dictionary[key] = _check_multiple_attributes( + metadata_dictionary[key], cube_names_dictionary[key + "_names"] ) - _write_metadata_file(writable_attribution, data_filepath, "attribution") - if len(restrictions) > 0: - writable_restrictions = _check_multiple_attributes( - restrictions, restrictions_names - ) - _write_metadata_file(writable_restrictions, data_filepath, "restrictions") + # write the metadata + _write_metadata_file(metadata_dictionary[key], data_filepath, key) def _check_multiple_attributes(attribute_list, cube_names): From 9abe77c8c632b3c6237420d3091c01499c1af591 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 11:54:16 +0100 Subject: [PATCH 52/80] Change save behaviour to add newlines --- lib/ants/io/save.py | 3 ++- lib/ants/tests/io/save/test__write_metadata.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 0e80db4..681e593 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -381,7 +381,8 @@ def _check_multiple_attributes(attribute_list, cube_names): # if they are not the same, add the cube name concatenated_attribute = [] for attribute, name in zip(attribute_list, cube_names, strict=True): - concatenated_attribute.append(name + " = " + attribute) + concatenated_attribute.append(name + " = " + attribute + "\n") + print("concatenated_attribute", concatenated_attribute) return concatenated_attribute diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__write_metadata.py index 79325d7..d556bd6 100644 --- a/lib/ants/tests/io/save/test__write_metadata.py +++ b/lib/ants/tests/io/save/test__write_metadata.py @@ -91,9 +91,9 @@ def test_multiple_cubes(tmp_path): with open(expected_filename, "r") as file: actual_license = file.read() expected_license = ( - "the first cube = This is a cube's license. the second cube = " - "This is another cube's license. the third cube = This is a third cube's " - "license. " + "the first cube = This is a cube's license. \nthe second cube = " + "This is another cube's license. \nthe third cube = This is a third cube's " + "license. \n" ) assert actual_license == expected_license From df0839195281beaccd82ae90ae09863502bda111 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 11:57:24 +0100 Subject: [PATCH 53/80] Rename test__write_metadata.py to test__check_and_sort_metadata_attributes.py --- ...te_metadata.py => test__check_and_sort_metadata_attributes.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/ants/tests/io/save/{test__write_metadata.py => test__check_and_sort_metadata_attributes.py} (100%) diff --git a/lib/ants/tests/io/save/test__write_metadata.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py similarity index 100% rename from lib/ants/tests/io/save/test__write_metadata.py rename to lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py From ccce4db7d2608c58d4d1533d54a8da2178b96e65 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 30 Jun 2026 14:20:34 +0100 Subject: [PATCH 54/80] Clean up unittests --- .../save/test__check_and_sort_metadata_attributes.py | 10 ---------- .../tests/io/save/test__check_multiple_attributes.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py index d556bd6..d738ffb 100644 --- a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -6,13 +6,9 @@ from unittest import mock import ants.tests.stock as stock -import pytest from ants.io.save import _check_and_sort_metadata_attributes -@pytest.mark.filterwarnings( - "ignore:license has been written to sidecar file /var/tmp/:UserWarning" -) def test_license_attribute_written(tmp_path): """Tests that a cube with a license is written out.""" cube = stock.geodetic(shape=(2, 2)) @@ -27,9 +23,6 @@ def test_license_attribute_written(tmp_path): assert actual_license == license -@pytest.mark.filterwarnings( - "ignore:license has been written to sidecar file /var/tmp/:UserWarning" -) def test_loaded_license_written(tmp_path): """Tests that a cube with a license that resembles the format of a longer license written in, is written out correctly.""" @@ -65,9 +58,6 @@ def test_log_output(caplog): assert expected_message in caplog.text -@pytest.mark.filterwarnings( - "ignore:license has been written to sidecar file /var/tmp/:UserWarning" -) def test_multiple_cubes(tmp_path): """Test that multiple_cubes with attributes is written out correctly.""" # creating three cubes with different license attributes diff --git a/lib/ants/tests/io/save/test__check_multiple_attributes.py b/lib/ants/tests/io/save/test__check_multiple_attributes.py index ae534ed..1d37924 100644 --- a/lib/ants/tests/io/save/test__check_multiple_attributes.py +++ b/lib/ants/tests/io/save/test__check_multiple_attributes.py @@ -40,7 +40,7 @@ def test_all_elements_in_list_same(): def test_different_elements_have_cube_names(): """Tests that when multiple attributes are given, cube names are included.""" - expected = ["cube1 = license 1. ", "cube2 = license 2. "] + expected = ["cube1 = license 1. \n", "cube2 = license 2. \n"] licenses = ["license 1. ", "license 2. "] cube_names = ["cube1", "cube2"] actual = _check_multiple_attributes(licenses, cube_names) From 84362b5303bf06da01114aba4f994a75e1734f87 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 1 Jul 2026 10:11:49 +0100 Subject: [PATCH 55/80] Add support for other attributes to be written out --- lib/ants/io/save.py | 9 ++++++++- .../io/save/test__check_and_sort_metadata_attributes.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 681e593..2be35aa 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -346,7 +346,14 @@ def _check_and_sort_metadata_attributes(cubes, data_filepath): # a dictionary to keep track of which cubes have metadata cube_names_dictionary = {} # a list of approved attributes that can be saved - attributes_to_save = ["license", "attribution", "restrictions"] + attributes_to_save = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] for cube in cubes: for key, value in cube.attributes.items(): # check the attribute is one we want to save diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py index d738ffb..78bc80e 100644 --- a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -46,10 +46,10 @@ def test_loaded_license_written(tmp_path): def test_log_output(caplog): """Tests that the logger gives the correct output when a file is written out.""" cube = stock.geodetic(shape=(2, 2)) - attribution = "This data came from an institution. " - cube.attributes["attribution"] = attribution + institution = "This data came from Unseen University. " + cube.attributes["institution"] = institution expected_message = ( - "attribution has been written to sidecar file filename.attribution" + "institution has been written to sidecar file filename.institution" ) # mocks out the opening of files, so no file is created with mock.patch("builtins.open"): From cc84db7b71950762e1854594ca9f5ed797e28202 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 1 Jul 2026 10:14:49 +0100 Subject: [PATCH 56/80] Change test placeholder reference --- .../tests/io/save/test__check_and_sort_metadata_attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py index 78bc80e..cac92af 100644 --- a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -46,7 +46,7 @@ def test_loaded_license_written(tmp_path): def test_log_output(caplog): """Tests that the logger gives the correct output when a file is written out.""" cube = stock.geodetic(shape=(2, 2)) - institution = "This data came from Unseen University. " + institution = "This data came from University Blah. " cube.attributes["institution"] = institution expected_message = ( "institution has been written to sidecar file filename.institution" From ae88f5795c9868a835d70f099bc401b1ffa86cc4 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 13 Jul 2026 11:36:53 +0100 Subject: [PATCH 57/80] Update comment --- lib/ants/io/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 2be35aa..a29470c 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -365,7 +365,7 @@ def _check_and_sort_metadata_attributes(cubes, data_filepath): metadata_dictionary[key] = [value] cube_names_dictionary[key + "_names"] = [cube.name()] for key, value in metadata_dictionary.items(): - # sort the metadata ready to save + # Update the metadata ready to save metadata_dictionary[key] = _check_multiple_attributes( metadata_dictionary[key], cube_names_dictionary[key + "_names"] ) From a159bc7696d2535924a97963ac7adaa0c3a9db4c Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 13 Jul 2026 11:38:26 +0100 Subject: [PATCH 58/80] Remove print statement --- lib/ants/io/save.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index a29470c..eeae8df 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -389,7 +389,6 @@ def _check_multiple_attributes(attribute_list, cube_names): concatenated_attribute = [] for attribute, name in zip(attribute_list, cube_names, strict=True): concatenated_attribute.append(name + " = " + attribute + "\n") - print("concatenated_attribute", concatenated_attribute) return concatenated_attribute From 7df8ca4755133a904a206482984bee60cf9e9b0f Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 13 Jul 2026 11:42:07 +0100 Subject: [PATCH 59/80] Update spelling --- lib/ants/io/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index eeae8df..e1be662 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -399,7 +399,7 @@ def _write_metadata_file(metadata, filename, attribute_name): appended to it. """ filepath = str(filename) + "." + attribute_name - # Order metadata to be in one list, if metadata contains list of list - possible in + # Order metadata to be in one list, if metadata contains list of lists - possible in # cases where metadata is being read in if any(isinstance(element, list) for element in metadata): metadata = np.concatenate(metadata).tolist() From d0478ec4cc09ecad3baf308ded6238fe11c48573 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Tue, 14 Jul 2026 09:48:56 +0100 Subject: [PATCH 60/80] remove merging artifacts --- lib/ants/cli/ancil_fill_n_merge.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index dede518..59c7efd 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -104,11 +104,8 @@ def main( end, netcdf_only, search_method, -<<<<<<< main blending_distance, -======= ignore_metadata_files, ->>>>>>> load_save_feature_branch ): """ Perform merge and fill operation on the provided sources. @@ -159,17 +156,14 @@ def main( search_method : :obj:`str` Select the search method to be used when filling missing points. The methods currently supported are "spiral" and "kdtree". -<<<<<<< main blending_distance : float Distance over which blending between the primary and alternate sources is applied. Note that this is in units of grid cells, not a physical distance. If ``None``, no blending is applied, and there will be a hard edge between the two sources. -======= ignore_metadata_files : :obj:`bool`, optional When set to True, files containing metadata will not be loaded alongside data and added as attributes to the cube. ->>>>>>> load_save_feature_branch Returns ------- @@ -289,11 +283,8 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, -<<<<<<< main blending_distance=args.blending_distance, -======= ignore_metadata_files=args.ignore_metadata_files, ->>>>>>> load_save_feature_branch ) From f3b85fb5113c827101cd3a52d711c2d49e27b63b Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 15 Jul 2026 10:14:50 +0100 Subject: [PATCH 61/80] Add fix for netcdf saving error --- lib/ants/io/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 1c393fc..9f63d8b 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -475,7 +475,7 @@ def _retrieve_metadata(self, metadata_files, cube): ) else: open_file = open(metadata_file, "r") - metadata = open_file.readlines() + metadata = open_file.read() open_file.close() cube.attributes[attribute_name] = metadata From e3aea7e98588fe5945c5ff39af438a18f10841f3 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 15 Jul 2026 11:35:21 +0100 Subject: [PATCH 62/80] Update unittests with new load format --- lib/ants/tests/io/load/test_CallbackMetadata.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 892b8ac..c7e2619 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -23,12 +23,10 @@ def test_metadata_files_added_to_attributes(tmp_path): It should be preserved and added to the cube when loaded. """ # How the text should look while stored in an array - loaded_license = [ - "This is the license of the cube.\n", - "\n", - " It should be preserved and added to the cube when loaded.\n", - " ", - ] + loaded_license = ( + "This is the license of the cube.\n\n It should be preserved" + " and added to the cube when loaded.\n " + ) test_cube = ants.tests.stock.geodetic(shape=(2, 2)) temporary_cube_path = tmp_path / "cube_attribute.pp" iris.save(test_cube, str(temporary_cube_path)) @@ -166,7 +164,7 @@ def test_misspelt_license_added(tmp_path): with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) loaded_cube = ants.io.load.load_cube(temporary_cube_path) - assert loaded_cube.attributes["license"] == ["a license"] + assert loaded_cube.attributes["license"] == "a license" def test_invalid_metadata_name(tmp_path): From fcf45a41a80e4efaaeb56b102bc609eb65f163c4 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 20 Jul 2026 12:40:04 +0100 Subject: [PATCH 63/80] Cleanup and add fix for using constraints --- lib/ants/io/load.py | 2 ++ rose-stem/flow.cylc | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9f63d8b..87e9103 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -404,6 +404,8 @@ def _add_callback(callback, *args, **kwargs): args = list(args) if len(args) == 1: kwargs["callback"] = callback + elif len(args) == 2: + args.append(callback) elif len(args) == 3: args[2] = callback args = tuple(args) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index bb3cad4..1846e8a 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -264,7 +264,6 @@ fi [[[environment]]] ROSE_TASK_APP = rose_ana ROSE_APP_OPT_CONF_KEYS = general_regrid - ANTS_KGO_DIRECTORY_OVERRIDE=/home/users/theo.geddes/cylc-run/dev-ants-core/run98/share/data/ [[rose_ana_general_regrid]] inherit = ROSE_ANA From d9294f8179bd134c46443abba4ca48f0973a48ad Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 22 Jul 2026 09:44:04 +0100 Subject: [PATCH 64/80] Add tests for _add_callback() --- lib/ants/tests/io/load/test__add_callback.py | 62 ++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 lib/ants/tests/io/load/test__add_callback.py diff --git a/lib/ants/tests/io/load/test__add_callback.py b/lib/ants/tests/io/load/test__add_callback.py new file mode 100644 index 0000000..b03a154 --- /dev/null +++ b/lib/ants/tests/io/load/test__add_callback.py @@ -0,0 +1,62 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +"""Tests that aren't suited to being testsed with end-to-end functionality in +test__CallbackMetadata.py.""" + +from ants.io.load import _add_callback, _CallbackMetadata + + +def test_one_argument_input(): + """Tests that the arguments are correctly returned when given one positional + argument.""" + callback = _CallbackMetadata(None) + source = "filepath" + expected_args = (source,) + expected_kwargs = {"callback": callback} + actual_args, actual_kwargs = _add_callback(callback, source) + assert actual_args == expected_args + assert actual_kwargs == expected_kwargs + + +def test_two_argument_input(): + """Tests that the arguments are correctly returned when given two positional + arguments.""" + callback = _CallbackMetadata(None) + source = "filepath" + constraint = "a test constraint" + expected_args = (source, constraint, callback) + # this should return an empty dictionary of kwargs + actual_args, _empty_kwargs = _add_callback(callback, source, constraint) + assert actual_args == expected_args + assert _empty_kwargs == {} + + +def test_three_argument_input(): + """Tests that the arguments are correctly returned when given three positional + arguments.""" + user_callback = "a callback that will be wrapped in _CallbackMetadata" + callback = _CallbackMetadata(user_callback) + source = "filepath" + constraint = "a test constraint" + expected_args = (source, constraint, callback) + # this should return an empty dictionary of kwargs + actual_args, _empty_kwargs = _add_callback( + callback, source, constraint, user_callback + ) + assert actual_args == expected_args + assert _empty_kwargs == {} + + +def test_kwargs_input(): + """Tests that arguments are correctly returned when given keyword arguments.""" + callback = _CallbackMetadata(None) + source = "filepath" + expected_args = (source,) + constraint = "a test constraint" + # user callback will always be removed if passed in as a kwarg before this function + expected_kwargs = {"constraint": constraint, "callback": callback} + actual_args, actual_kwargs = _add_callback(callback, source, constraint=constraint) + assert actual_args == expected_args + assert actual_kwargs == expected_kwargs From 5676e054707ba672ab8d321a9a5c8cdefcab5548 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 10 Aug 2026 09:21:56 +0100 Subject: [PATCH 65/80] Add copy metadata attributes --- lib/ants/utils/cube.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 076e919..8638052 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1139,6 +1139,54 @@ def inherit_metadata(source, reference): source.attributes["grid_staggering"] = reference.attributes["grid_staggering"] +def copy_metadata_attributes( + source, + reference, + approved_metadata=[ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ], +): + """ + Inherit cube metadata attributes from a provided reference. + + In-place operation on source cube. + + Metadata attributes from the refrence are added into the source cube. This is done + in the manner of `attribute: cube-name = metadata content` for tracebaility and + allowing for the stacking of metadata. + + Parameters + ---------- + source : :class:`iris.cube.Cube` + Source to have its metadata update. + reference : :class:`iris.cube.Cube` + Reference which defines the metadata to inherit from. + """ + + for attribute in approved_metadata: + if attribute in reference.attributes: + if attribute in source.attributes: + if source.attributes[attribute] != reference.attributes[attribute]: + source.attributes[attribute] = ( + source.name() + + " = " + + source.attributes[attribute] + + "\n" + + reference.name() + + " = " + + reference.attributes[attribute] + ) + else: + source.attributes[attribute] = ( + reference.name() + " = " + reference.attributes[attribute] + ) + + def set_crs(cube, crs=None): """ Set cube coordinate system. From 209b4c6f69be4dcdf49d7dfd0c9b662400086a98 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 10 Aug 2026 09:58:18 +0100 Subject: [PATCH 66/80] Add tests for copying metadata --- .../cube/test_copy_metadata_attributes.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 lib/ants/tests/utils/cube/test_copy_metadata_attributes.py diff --git a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py new file mode 100644 index 0000000..604e275 --- /dev/null +++ b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py @@ -0,0 +1,69 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +import pytest +from ants.tests import stock +from ants.utils.cube import copy_metadata_attributes + + +def test_adding_new_attribute(): + """Tests that a new attribute will be added to the source cube if it doesn't exist + already.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["license"] = "a test license" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["license"] == "test cube = a test license" + + +def test_modifying_existing_attribute(): + """Tests that an existing attribute will be modified on the source cube.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["references"] = "a test reference" + source.rename("source cube") + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["references"] = "another different test reference" + reference.rename("reference cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert ( + source.attributes["references"] + == "source cube = a test reference\n" + + "reference cube = another different test reference" + ) + + +def test_no_reference_attribute(): + """Tests that a source cube attribute with no reference will be left unchanged.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["restrictions"] == "a test restriction" + + +def test_non_standard_attribute(): + """Tests that an attribute not in the allowed list is not copied over.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["a value not in the allowed list"] = "not in the list" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + expected_msg = "a value not in the allowed list" + with pytest.raises(KeyError, match=expected_msg): + # Check the attribute hasn't been copied over + source.attributes["a value not in the allowed list"] From c0e3128740ef64df17fe319c499e17753a7dfac7 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 10 Aug 2026 10:13:49 +0100 Subject: [PATCH 67/80] Add check for licence spelling when saving --- lib/ants/io/save.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index ec01a83..bba5c11 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -353,6 +353,13 @@ def _check_and_sort_metadata_attributes(cubes, data_filepath): for cube in cubes: for key, value in cube.attributes.items(): # check the attribute is one we want to save + if key == "licence": + warnings.warn( + "The attribute name licence has been changed to " + "'license', in line with ANTS working practices.", + category=UserWarning, + ) + key = "license" if key in attributes_to_save: if key in metadata_dictionary: metadata_dictionary[key].append(value) From 37ccb1398c849f5fccbc41691ce174dfded853fb Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 10 Aug 2026 10:35:32 +0100 Subject: [PATCH 68/80] Add test for licence key change --- ...est__check_and_sort_metadata_attributes.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py index cac92af..e2190a5 100644 --- a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -6,6 +6,7 @@ from unittest import mock import ants.tests.stock as stock +import pytest from ants.io.save import _check_and_sort_metadata_attributes @@ -23,6 +24,24 @@ def test_license_attribute_written(tmp_path): assert actual_license == license +@pytest.mark.filterwarnings( + "ignore:The attribute name licence has been changed to 'license', in line with " + "ANTS working practices.:UserWarning" +) +def test_licence_attribute_written(tmp_path): + """Tests that a cube with a licence is changed and written out.""" + cube = stock.geodetic(shape=(2, 2)) + licence = "This is a cube's license. " + cube.attributes["licence"] = licence + cube.rename("licence test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == licence + + def test_loaded_license_written(tmp_path): """Tests that a cube with a license that resembles the format of a longer license written in, is written out correctly.""" From 1a95458e8b20e0c83a0e280cb9be0f502b43bc2e Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 10:49:56 +0100 Subject: [PATCH 69/80] Update attribute name --- lib/ants/utils/cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 8638052..d46c3f2 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1142,7 +1142,7 @@ def inherit_metadata(source, reference): def copy_metadata_attributes( source, reference, - approved_metadata=[ + metadata_to_copy=[ "license", "attribution", "restrictions", @@ -1168,7 +1168,7 @@ def copy_metadata_attributes( Reference which defines the metadata to inherit from. """ - for attribute in approved_metadata: + for attribute in metadata_to_copy: if attribute in reference.attributes: if attribute in source.attributes: if source.attributes[attribute] != reference.attributes[attribute]: From 7cff6841212dabfd7675649bba71e67ad2c15a49 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 10:59:24 +0100 Subject: [PATCH 70/80] add test for changing metadata list --- .../utils/cube/test_copy_metadata_attributes.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py index 604e275..d06222e 100644 --- a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py +++ b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py @@ -67,3 +67,18 @@ def test_non_standard_attribute(): with pytest.raises(KeyError, match=expected_msg): # Check the attribute hasn't been copied over source.attributes["a value not in the allowed list"] + + +def test_different_attribute_list(): + """Tests that a different attribute list will copy over those attributes.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["test-attribute"] = "an attribute not in the standard list." + reference.rename("test cube") + copy_metadata_attributes(source, reference, metadata_to_copy=["test-attribute"]) + assert ( + source.attributes["test-attribute"] + == "test cube = an attribute not in the standard list." + ) From 8910d4a6c80b0af8156b9b205f8b7b2190cd130f Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 11:06:46 +0100 Subject: [PATCH 71/80] Add parameter to docstring --- lib/ants/utils/cube.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index d46c3f2..3d802c4 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1166,6 +1166,8 @@ def copy_metadata_attributes( Source to have its metadata update. reference : :class:`iris.cube.Cube` Reference which defines the metadata to inherit from. + metadata_to_copy : list + A list of metadata attribute keys to copy to the source cube. """ for attribute in metadata_to_copy: From 39e5229ac0612a1f7baaf2da613968e82a86f2d2 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 11:49:02 +0100 Subject: [PATCH 72/80] Add logic comments --- lib/ants/utils/cube.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 3d802c4..ba0aaa5 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1171,8 +1171,13 @@ def copy_metadata_attributes( """ for attribute in metadata_to_copy: + # Only copy an attribute if it exists in the reference cube. if attribute in reference.attributes: + # If the attribute already exists in the source cube if attribute in source.attributes: + # If the attributes are the same then nothing needs to be done. + # If the attributes are not the same then both names must be prepended + # to ensure that the metadata is traceable. if source.attributes[attribute] != reference.attributes[attribute]: source.attributes[attribute] = ( source.name() @@ -1184,6 +1189,7 @@ def copy_metadata_attributes( + reference.attributes[attribute] ) else: + # If the attribute does not exist in the source cube source.attributes[attribute] = ( reference.name() + " = " + reference.attributes[attribute] ) From 288ef806dda1ad78438dcc8733b4a2c41c85fed1 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 11:50:04 +0100 Subject: [PATCH 73/80] Add logic comments --- lib/ants/utils/cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index ba0aaa5..d98884f 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1173,7 +1173,7 @@ def copy_metadata_attributes( for attribute in metadata_to_copy: # Only copy an attribute if it exists in the reference cube. if attribute in reference.attributes: - # If the attribute already exists in the source cube + # If the attribute already exists in the source cube. if attribute in source.attributes: # If the attributes are the same then nothing needs to be done. # If the attributes are not the same then both names must be prepended @@ -1189,7 +1189,7 @@ def copy_metadata_attributes( + reference.attributes[attribute] ) else: - # If the attribute does not exist in the source cube + # If the attribute does not exist in the source cube.\ source.attributes[attribute] = ( reference.name() + " = " + reference.attributes[attribute] ) From bedd16c7fed80013fe4ab90dd10964ea7660288f Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 17 Aug 2026 13:37:34 +0100 Subject: [PATCH 74/80] add whitespace stripping --- .../cube/test_copy_metadata_attributes.py | 28 +++++++++++++++++++ lib/ants/utils/cube.py | 5 +++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py index d06222e..8152be8 100644 --- a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py +++ b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py @@ -82,3 +82,31 @@ def test_different_attribute_list(): source.attributes["test-attribute"] == "test cube = an attribute not in the standard list." ) + + +def test_same_attributes(): + """Tests that if both cubes have the same attribute, nothing happens.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction" + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" + + +def test_different_whitespace(): + """Tests that attributes with different whitespace will be registered as equal.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction " + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index d98884f..32ddf87 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1175,10 +1175,13 @@ def copy_metadata_attributes( if attribute in reference.attributes: # If the attribute already exists in the source cube. if attribute in source.attributes: + # Remove whitepace before comparisons. + source_compare = " ".join(source.attributes[attribute].split()) + reference_compare = " ".join(reference.attributes[attribute].split()) # If the attributes are the same then nothing needs to be done. # If the attributes are not the same then both names must be prepended # to ensure that the metadata is traceable. - if source.attributes[attribute] != reference.attributes[attribute]: + if source_compare != reference_compare: source.attributes[attribute] = ( source.name() + " = " From d1163ea85561221571254d42e58f19b32c8fafef Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 20 Aug 2026 09:18:49 +0100 Subject: [PATCH 75/80] Update lib/ants/utils/cube.py Co-authored-by: Andrew Clark <2562650+arjclark@users.noreply.github.com> Signed-off-by: Theo Geddes --- lib/ants/utils/cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 32ddf87..033af48 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1192,7 +1192,7 @@ def copy_metadata_attributes( + reference.attributes[attribute] ) else: - # If the attribute does not exist in the source cube.\ + # If the attribute does not exist in the source cube. source.attributes[attribute] = ( reference.name() + " = " + reference.attributes[attribute] ) From 8b299c8afc27600285b68bbbb4810e933a2b2879 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 7 Sep 2026 12:53:37 +0100 Subject: [PATCH 76/80] #32: Add more accepted metadata attributes to load --- lib/ants/io/load.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 87e9103..766e271 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -450,7 +450,14 @@ def _retrieve_metadata(self, metadata_files, cube): The cube being loaded. """ - valid_metadata_names = ["license", "attribution", "restrictions"] + valid_metadata_names = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] other_license = "licence" for metadata_file in metadata_files: file_name_splits = str(metadata_file).split(".") From f1f4f3e1e4c343e6b5262185a29b238f60c9e554 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 7 Sep 2026 12:54:32 +0100 Subject: [PATCH 77/80] #32: Add documentation on handling metadata --- docs/source/tutorial_metadata.rst | 49 +++++++++++++++++++++++++++++++ docs/source/tutorials.rst | 1 + 2 files changed, 50 insertions(+) create mode 100644 docs/source/tutorial_metadata.rst diff --git a/docs/source/tutorial_metadata.rst b/docs/source/tutorial_metadata.rst new file mode 100644 index 0000000..a2e2c4d --- /dev/null +++ b/docs/source/tutorial_metadata.rst @@ -0,0 +1,49 @@ +.. meta:: + :description lang=en: Tutorial on managing metadata + :keywords: metadata, license, licensing, attribution, references, development, tutorial + :property=og:locale: en_GB + +.. include:: common.txt + + +Managing Metadata +================= + +Metadata in this tutorial refers to information about the data in a file, that does not +directly affect that data. For example, licensing information or attributions. + +Some ancillary file formats cannot include metadata. Metadata can be very important to +keep alongside the data as often data will have rome form of requirement or restriction +on it. Because of this, ANTS can handle in external metadata files, provided they match +the naming convention of `filename.attribute.accepted-metadata`. The current accepted +metadata attributes are "license", "attribution", "restrictions", "institution", +"acknowledgement", and "references". + + +.. note:: + Any attributes or files referencing licensing should use the 'license' spelling for + consistency. + +Loading Metadata +---------------- + +ANTS can load in external metadata files or 'sidecar' files, if they are kept in the +same directory as the data files, and have the same name (including the extension). +Metadata can be loaded alongside all files, including NetCDF files, however ANTS will +not allow you to load a file with a metadata attribute and a sidecar file with the same +attribute. This is to prevent unintentional loss or overwriting of metadata information. + +.. note:: + When using wildcards for loading with sidecar files, add the file extension to the + end to prevent issues with ANTS attempting to load the sidecar files. E.g. + `data*.pp` rather than `data*`. + + +Saving Metadata +--------------- + +Metadata will only be saved to a sidecar file, if the ancil loader is used. This is +because NetCDF files will include the attributes within the file. + +The sidecar files produced by ANTS follow the same naming covenstions at those loaded in +`filename.attribute.accepted-metadata`. Only accepted metadata will be written out. diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index ba69a82..51190e8 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -14,3 +14,4 @@ Tutorials tutorial_KGO.rst tutorial_sources.rst tutorial_rose_stem.rst + tutorial_metadata.rst From fd5cec43e6730036ad680f129c2c884f570c38e0 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 7 Sep 2026 12:58:42 +0100 Subject: [PATCH 78/80] #33: add metadata rose stem --- rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf | 2 -- .../rose_ana/opt/rose-app-general_regrid_metadata.conf | 3 +++ rose-stem/flow.cylc | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf diff --git a/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf b/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf index 6f7127b..487f9de 100644 --- a/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf +++ b/rose-stem/app/rose_ana/opt/rose-app-general_regrid.conf @@ -29,5 +29,3 @@ filelist=ancil_general_regrid_grid_to_variable_resolution_grid_split1 =ancil_general_regrid_grid_to_n48e_namelist_split0.nc =ancil_general_regrid_3d_to_3d_split0.nc =ancil_general_regrid_3d_to_3d_with_extrapolation_split0.nc - =ancil_general_regrid_metadata - =ancil_general_regrid_metadata.nc diff --git a/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf new file mode 100644 index 0000000..e9b1eea --- /dev/null +++ b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf @@ -0,0 +1,3 @@ +[env] +filelist=ancil_general_regrid_metadata + =ancil_general_regrid_metadata.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 1846e8a..3ba6ba1 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -259,6 +259,14 @@ fi inherit = ANTS_CORE, ANCIL_GENERAL_REGRID, LARGE script = rose task-run --app-key=ancil_general_regrid -O metadata + [[rose_ana_general_regrid_metadata]] + inherit = ROSE_ANA + [[[environment]]] + ROSE_TASK_APP = rose_ana + ROSE_APP_OPT_CONF_KEYS = general_regrid_metadata + ANTS_KGO_DIRECTORY_OVERRIDE = /data/users/theo.geddes/ants-source/KGO/ + + [[rose_ana_general_regrid]] inherit = ROSE_ANA [[[environment]]] From 64152eb34b6f502ce7053b43a750fcc1d641b32e Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 7 Sep 2026 14:08:26 +0100 Subject: [PATCH 79/80] #32: Add documentation on turning off metadata load --- docs/source/tutorial_metadata.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/tutorial_metadata.rst b/docs/source/tutorial_metadata.rst index a2e2c4d..5ae1a73 100644 --- a/docs/source/tutorial_metadata.rst +++ b/docs/source/tutorial_metadata.rst @@ -19,6 +19,12 @@ the naming convention of `filename.attribute.accepted-metadata`. The current acc metadata attributes are "license", "attribution", "restrictions", "institution", "acknowledgement", and "references". +This functionality can be turned off through the use of the argument +``--ignore-metadata-files`` on the command line or when calling the ANTS load: + +.. code-block:: python + cube = ants.io.load('data', ignore_metadata_files=True) + .. note:: Any attributes or files referencing licensing should use the 'license' spelling for From 1239cf3bca57e33288e73bde5d86c63d3671d7a6 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Mon, 7 Sep 2026 14:15:39 +0100 Subject: [PATCH 80/80] #32: Add whitepsace --- docs/source/tutorial_metadata.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/tutorial_metadata.rst b/docs/source/tutorial_metadata.rst index 5ae1a73..01897d8 100644 --- a/docs/source/tutorial_metadata.rst +++ b/docs/source/tutorial_metadata.rst @@ -23,6 +23,7 @@ This functionality can be turned off through the use of the argument ``--ignore-metadata-files`` on the command line or when calling the ANTS load: .. code-block:: python + cube = ants.io.load('data', ignore_metadata_files=True)