diff --git a/docs/source/tutorial_metadata.rst b/docs/source/tutorial_metadata.rst new file mode 100644 index 0000000..01897d8 --- /dev/null +++ b/docs/source/tutorial_metadata.rst @@ -0,0 +1,56 @@ +.. meta:: + :description lang=en: Tutorial on managing metadata + :keywords: metadata, license, licensing, attribution, references, development, tutorial + :property=og:locale: en_GB + +.. include:: common.txt + + +Managing Metadata +================= + +Metadata in this tutorial refers to information about the data in a file, that does not +directly affect that data. For example, licensing information or attributions. + +Some ancillary file formats cannot include metadata. Metadata can be very important to +keep alongside the data as often data will have rome form of requirement or restriction +on it. Because of this, ANTS can handle in external metadata files, provided they match +the naming convention of `filename.attribute.accepted-metadata`. The current accepted +metadata attributes are "license", "attribution", "restrictions", "institution", +"acknowledgement", and "references". + +This functionality can be turned off through the use of the argument +``--ignore-metadata-files`` on the command line or when calling the ANTS load: + +.. code-block:: python + + cube = ants.io.load('data', ignore_metadata_files=True) + + +.. note:: + Any attributes or files referencing licensing should use the 'license' spelling for + consistency. + +Loading Metadata +---------------- + +ANTS can load in external metadata files or 'sidecar' files, if they are kept in the +same directory as the data files, and have the same name (including the extension). +Metadata can be loaded alongside all files, including NetCDF files, however ANTS will +not allow you to load a file with a metadata attribute and a sidecar file with the same +attribute. This is to prevent unintentional loss or overwriting of metadata information. + +.. note:: + When using wildcards for loading with sidecar files, add the file extension to the + end to prevent issues with ANTS attempting to load the sidecar files. E.g. + `data*.pp` rather than `data*`. + + +Saving Metadata +--------------- + +Metadata will only be saved to a sidecar file, if the ancil loader is used. This is +because NetCDF files will include the attributes within the file. + +The sidecar files produced by ANTS follow the same naming covenstions at those loaded in +`filename.attribute.accepted-metadata`. Only accepted metadata will be written out. diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst index ba69a82..51190e8 100644 --- a/docs/source/tutorials.rst +++ b/docs/source/tutorials.rst @@ -14,3 +14,4 @@ Tutorials tutorial_KGO.rst tutorial_sources.rst tutorial_rose_stem.rst + tutorial_metadata.rst diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 60deaa1..2d24b97 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,16 @@ import iris -def load_data(source): +def load_data(source, ignore_metadata_files): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source) + cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only): +def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files): """ Convert specified source file to an ancillary. @@ -66,13 +66,17 @@ def main(source_path, output_path, grid_staggering, netcdf_only): written to an ancillary. """ - source_cubes = load_data(source_path) + source_cubes = load_data(source_path, ignore_metadata_files) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering if not netcdf_only: - save.ancil(source_cubes, output_path) + save.ancil( + source_cubes, + output_path, + ignore_writing_metadata_files=ignore_metadata_files, + ) save.netcdf(source_cubes, output_path) return source_cubes @@ -98,6 +102,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, + args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index cf83a4d..59c7efd 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,6 +25,7 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): """ Load the necessary data for performing a merge and fill operation. @@ -45,6 +46,9 @@ def load_data( Datetime to start the processing. end: :obj:`datetime`, optional Datetime to end the processing. + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns @@ -56,12 +60,16 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source) + primary_cubes = ants.io.load.load( + primary_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source) + alternate_cubes = ants.io.load.load( + alternate_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -97,6 +105,7 @@ def main( netcdf_only, search_method, blending_distance, + ignore_metadata_files, ): """ Perform merge and fill operation on the provided sources. @@ -152,6 +161,9 @@ def main( is applied. Note that this is in units of grid cells, not a physical distance. If ``None``, no blending is applied, and there will be a hard edge between the two sources. + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns ------- @@ -171,6 +183,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files, ) result = primary_cubes @@ -182,7 +195,7 @@ def main( ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method) if not netcdf_only: - save.ancil(result, output) + save.ancil(result, output, ignore_writing_metadata_files=ignore_metadata_files) save.netcdf(result, output) return result @@ -271,6 +284,7 @@ def cli_interface(): netcdf_only=args.netcdf_only, search_method=args.search_method, blending_distance=args.blending_distance, + ignore_metadata_files=args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 7f6bd86..355226a 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -42,12 +42,17 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load( + source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid) + target_cube = ants.io.load.load_grid( + target_grid, ignore_metadata_files=ignore_metadata_files + ) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold @@ -76,6 +81,7 @@ def main( save_ukca, netcdf_only, search_method, + ignore_metadata_files, ): """ General regrid application top level call function. @@ -118,6 +124,9 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. + ignore_metadata_files : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns ------- @@ -132,6 +141,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -149,7 +159,11 @@ def main( save.ukca_netcdf(regridded_cubes, output_path) else: if not netcdf_only: - save.ancil(regridded_cubes, output_path) + save.ancil( + regridded_cubes, + output_path, + ignore_writing_metadata_files=ignore_metadata_files, + ) save.netcdf(regridded_cubes, output_path) return regridded_cubes @@ -210,6 +224,7 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, + args.ignore_metadata_files, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 2a3a01c..e40cb0c 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. + --ignore-metadata-files Stops the application from + searching for metadata files + with the same name as the + source and loading them + alongside the source. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) + self.add_argument( + "--ignore-metadata-files", + action="store_true", + help="Stops the application from searching for metadata files with " + "the same name as the source and loading them alongside the source.", + required=False, + ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 9c53786..766e271 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -56,6 +56,7 @@ """ import copy +import glob import warnings from contextlib import contextmanager from functools import wraps @@ -231,18 +232,22 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask") + lbm = ants.io.load.load_cube( + filename, "land_binary_mask", ignore_metadata_files=True + ) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") + land_fraction = ants.io.load.load_cube( + filename, "vegetation_area_fraction", ignore_metadata_files=True + ) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename)[0] + cube = ants.io.load.load(filename, ignore_metadata_files=True)[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -355,6 +360,21 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) + ignore_metadata_files = False + if "ignore_metadata_files" in kwargs: + ignore_metadata_files = kwargs.pop("ignore_metadata_files") + if not ignore_metadata_files: + # Do the handling for each way a user callback can be passed in through + # iris + user_callback = None + if len(args) == 3: + user_callback = args[2] + else: + if "callback" in kwargs: + user_callback = kwargs.pop("callback") + args, kwargs = _add_callback( + _CallbackMetadata(user_callback), *args, **kwargs + ) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -370,6 +390,105 @@ def load_function(*args, **kwargs): return load_function +def _add_callback(callback, *args, **kwargs): + """ + Adds both the ants callback and the user provided callback (if any) to the + load. + + Parameters + ---------- + callback: :class:`_CallbackMetadata` + An object that will contain the ants call back and an attribute + with the user callback if applicable. + """ + args = list(args) + if len(args) == 1: + kwargs["callback"] = callback + elif len(args) == 2: + args.append(callback) + elif len(args) == 3: + args[2] = callback + args = tuple(args) + return args, kwargs + + +class _CallbackMetadata(object): + """Callback for collecting metadata from sidecar files. + + This callback will load additional metadata files with the naming convention: + filename.[license,attribution,restrictions] and append the contents of those files + to the cube attributes. + """ + + def __init__(self, user_callback): + self._user_callback = user_callback + + def __call__(self, cube, field, filename): + """ + The method that runs when Iris runs the callback. Collects the filenames and + will run the user callback + """ + if type(filename) is list: + filename = filename[0] + metadata_filenames = "".join([filename, ".*"]) + metadata_files = glob.glob(metadata_filenames) + if metadata_files != []: + self._retrieve_metadata(metadata_files, cube) + if self._user_callback is not None: + self._user_callback(cube, field, filename) + + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + valid_metadata_names = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] + other_license = "licence" + for metadata_file in metadata_files: + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + if attribute_name == other_license: + warnings.warn( + f"The attribute name {attribute_name} has been changed to " + "'license', in line with ANTS working practices.", + category=UserWarning, + ) + attribute_name = "license" + if attribute_name in cube.attributes: + raise AttributeError( + f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + if attribute_name not in valid_metadata_names: + warnings.warn( + f"Attribute {attribute_name} is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions.", + category=UserWarning, + ) + else: + open_file = open(metadata_file, "r") + metadata = open_file.read() + open_file.close() + cube.attributes[attribute_name] = metadata + + def load_cube(*args, **kwargs): """ Loads a single cube. diff --git a/lib/ants/io/save.py b/lib/ants/io/save.py index 9ebd7bf..3ded15b 100644 --- a/lib/ants/io/save.py +++ b/lib/ants/io/save.py @@ -20,12 +20,14 @@ specifying ``saver='ukca'`` (see :func:`ants.io.save.ukca_netcdf`). """ +import logging import os import sys import warnings import ants.utils.cube import iris +import numpy as np from ants.fileformats.ancil import _cubes_to_ancilfile, _mule_set_lbuser2 from ants.fileformats.netcdf.cf import ( _coerce_netcdf_classic_dtypes, @@ -34,7 +36,7 @@ from ants.fileformats.netcdf.ukca import LOCAL_ATTS, _ukca_conventions -def ancil(cubes, filename): +def ancil(cubes, filename, ignore_writing_metadata_files=False): """ Save one or more cubes to a F03 UM ancillary file. @@ -72,6 +74,9 @@ def ancil(cubes, filename): One or more cubes to be saved. filename : str The name of the F03 UM ancillary file, including any extension. + ignore_writing_metadata_files : bool + Determines whether attributes should be saved to a seperate metadata file. + Default setting is False, so will write out the metadata. Notes ----- @@ -91,6 +96,8 @@ def ancil(cubes, filename): raise ValueError("F03 UM ancillary files cannot be saved with a .nc extension.") cubes = ants.utils.cube.as_cubelist(cubes) + if not ignore_writing_metadata_files: + _check_and_sort_metadata_attributes(cubes, filename) ancilfile = _cubes_to_ancilfile(cubes) _mule_set_lbuser2(ancilfile) ancilfile.to_file(filename) @@ -319,3 +326,88 @@ def _update_history_cmd(cube): items[0] = os.path.basename(items[0]) items.append(f"({metadata})") if metadata else None ants.utils.cube.update_history(cubes, " ".join(items)) + + +def _check_and_sort_metadata_attributes(cubes, data_filepath): + """Checks for a license, attribution or restrictions in the metadata of the cubes, + and calls `_write_metadata_file` for each attribute. + Parameters + ---------- + cubes : :class:`iris.cube.Cube` or :class:`iris.cube.CubeList` + One or more cubes to be saved. + data_filepath : str + The name of the file where the data will be saved to. + """ + # a dictionary to contain all of the metadata to be saved + metadata_dictionary = {} + # a dictionary to keep track of which cubes have metadata + cube_names_dictionary = {} + # a list of approved attributes that can be saved + attributes_to_save = [ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ] + for cube in cubes: + for key, value in cube.attributes.items(): + # check the attribute is one we want to save + if key == "licence": + warnings.warn( + "The attribute name licence has been changed to " + "'license', in line with ANTS working practices.", + category=UserWarning, + ) + key = "license" + if key in attributes_to_save: + if key in metadata_dictionary: + metadata_dictionary[key].append(value) + cube_names_dictionary[key + "_names"].append(cube.name()) + else: + metadata_dictionary[key] = [value] + cube_names_dictionary[key + "_names"] = [cube.name()] + for key, value in metadata_dictionary.items(): + # Update the metadata ready to save + metadata_dictionary[key] = _check_multiple_attributes( + metadata_dictionary[key], cube_names_dictionary[key + "_names"] + ) + # write the metadata + _write_metadata_file(metadata_dictionary[key], data_filepath, key) + + +def _check_multiple_attributes(attribute_list, cube_names): + """Checks whether the attribute can be written out exactly as is, or if it has to be + pre-pended with the cube name.""" + # check if multiple things in list + if len(attribute_list) == 1: + return attribute_list + # check if attributes are the same + if len(set(attribute_list)) == 1: + return attribute_list[:1] + # check if there is only one cube + if len(cube_names) == 1: + return attribute_list + # if they are not the same, add the cube name + concatenated_attribute = [] + for attribute, name in zip(attribute_list, cube_names, strict=True): + concatenated_attribute.append(name + " = " + attribute + "\n") + return concatenated_attribute + + +def _write_metadata_file(metadata, filename, attribute_name): + """Takes a list of metadata and writes it to a file called + filename.. + If for any reason, the file to be written already exists, the new metadata will be + appended to it. + """ + filepath = str(filename) + "." + attribute_name + # Order metadata to be in one list, if metadata contains list of lists - possible in + # cases where metadata is being read in + if any(isinstance(element, list) for element in metadata): + metadata = np.concatenate(metadata).tolist() + with open(filepath, "a") as metadata_file: + metadata_file.writelines(metadata) + _LOGGER = logging.getLogger(__name__) + _LOGGER.info(f"{attribute_name} has been written to sidecar file {filepath}") diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 31105ab..50c11e0 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,6 +30,7 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): + """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -53,6 +54,7 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -69,6 +71,7 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, + ignore_metadata_files=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -150,6 +153,7 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): + """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -176,11 +180,13 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): + """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -207,6 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -265,3 +272,30 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() + + def test_set_ignore_metadata_files_flag(self): + "Tests that ignore_metadata_files will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-files", + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_files=True, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py new file mode 100644 index 0000000..c7e2619 --- /dev/null +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -0,0 +1,183 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +""" +Includes tests for end-to-end functionality of CallbackMetadata as well as calling the +class directly. +""" + +import unittest.mock as mock +import warnings + +import ants.io.load +import iris +import pytest + + +def test_metadata_files_added_to_attributes(tmp_path): + """Tests that metadata files are found and added to the cube's attributes.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + # How the text should look while stored in an array + loaded_license = ( + "This is the license of the cube.\n\n It should be preserved" + " and added to the cube when loaded.\n " + ) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_test_cube.attributes["license"] == loaded_license + + +def test_no_metadata_loaded(tmp_path): + """Tests that metadata files are not loaded when the option is turned off.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube( + temporary_cube_path, ignore_metadata_files=True + ) + with pytest.raises(KeyError): + loaded_test_cube.attributes["license"] + + +def test_user_callback_added(): + """Test that on ititialisation, the user's function will be set.""" + + def user_callback(cube, field, filename): + print("a user's callback, passed in") + + class_instance = ants.io.load._CallbackMetadata(user_callback) + assert class_instance._user_callback == user_callback + + +def test_args_parsed_correctly_with_kwargs(tmp_path): + """Tests that when passed a user callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, callback=mock_callback) + mock_callback.assert_called() + + +def test_args_parsed_correctly_with_positional_args(tmp_path): + """Tests that when passed a user callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, None, mock_callback) + mock_callback.assert_called() + + +def test__retrieve_metadata_correct_name(): + """Tests that correctly named metadata files are going to be read.""" + class_instance = ants.io.load._CallbackMetadata(None) + mock_file_open = mock.mock_open() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + path = ["fake-path/fake-cube.attribution"] + with mock.patch("builtins.open", mock_file_open): + class_instance._retrieve_metadata(path, test_cube) + mock_file_open.assert_called_once_with("fake-path/fake-cube.attribution", "r") + + +def test__retrieve_metadata_incorrect_name(): + """Tests that files without a valid metadata name won't be read.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + path = ["fake-path/fake-cube.pp"] + expected_message = ( + "Attribute pp is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions." + ) + with pytest.raises(UserWarning, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +def test_attribute_already_on_cube(): + """Tests that having both metadata attributes and sidecar files of the same type + will raise an attribute error.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes["attribution"] = "This is a attribution. " + path = ["fake-path/fake-cube.attribution"] + expected_message = ( + "The attribution is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +@pytest.mark.filterwarnings( + "ignore:The attribute name licence has been changed to 'license', in line with " + "ANTS working practices.:UserWarning" +) +def test_missplet_license_with_licensed_cube(): + """Tests that when given a sidecar file with a miss-spelt license and a cube with an + existing license, an error will be given.""" + class_instance = ants.io.load._CallbackMetadata(None) + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes["license"] = "This is a license. " + path = ["fake-path/fake-cube.licence"] + expected_message = ( + "The license is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) + with pytest.raises(AttributeError, match=expected_message): + class_instance._retrieve_metadata(path, test_cube) + + +def test_misspelt_license_added(tmp_path): + """Tests that a different spelling of license will add a license attribute.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.licence" + temporary_license_path.write_text(license_text, encoding="utf-8") + # ignore warning that will be raised + warning_message = ( + "The attribute name licence has been changed to 'license', in " + "line with ANTS working practices." + ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) + loaded_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_cube.attributes["license"] == "a license" + + +def test_invalid_metadata_name(tmp_path): + """Tests that an invalid metadata name will not be added as an attribute.""" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" + temporary_license_path.write_text(" ", encoding="utf-8") + + warning_message = ( + "Attribute invalid-name is not a valid metadata file name. " + "Accepted metadata names are license, attribution and restrictions." + ) + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) diff --git a/lib/ants/tests/io/load/test__add_callback.py b/lib/ants/tests/io/load/test__add_callback.py new file mode 100644 index 0000000..b03a154 --- /dev/null +++ b/lib/ants/tests/io/load/test__add_callback.py @@ -0,0 +1,62 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +"""Tests that aren't suited to being testsed with end-to-end functionality in +test__CallbackMetadata.py.""" + +from ants.io.load import _add_callback, _CallbackMetadata + + +def test_one_argument_input(): + """Tests that the arguments are correctly returned when given one positional + argument.""" + callback = _CallbackMetadata(None) + source = "filepath" + expected_args = (source,) + expected_kwargs = {"callback": callback} + actual_args, actual_kwargs = _add_callback(callback, source) + assert actual_args == expected_args + assert actual_kwargs == expected_kwargs + + +def test_two_argument_input(): + """Tests that the arguments are correctly returned when given two positional + arguments.""" + callback = _CallbackMetadata(None) + source = "filepath" + constraint = "a test constraint" + expected_args = (source, constraint, callback) + # this should return an empty dictionary of kwargs + actual_args, _empty_kwargs = _add_callback(callback, source, constraint) + assert actual_args == expected_args + assert _empty_kwargs == {} + + +def test_three_argument_input(): + """Tests that the arguments are correctly returned when given three positional + arguments.""" + user_callback = "a callback that will be wrapped in _CallbackMetadata" + callback = _CallbackMetadata(user_callback) + source = "filepath" + constraint = "a test constraint" + expected_args = (source, constraint, callback) + # this should return an empty dictionary of kwargs + actual_args, _empty_kwargs = _add_callback( + callback, source, constraint, user_callback + ) + assert actual_args == expected_args + assert _empty_kwargs == {} + + +def test_kwargs_input(): + """Tests that arguments are correctly returned when given keyword arguments.""" + callback = _CallbackMetadata(None) + source = "filepath" + expected_args = (source,) + constraint = "a test constraint" + # user callback will always be removed if passed in as a kwarg before this function + expected_kwargs = {"constraint": constraint, "callback": callback} + actual_args, actual_kwargs = _add_callback(callback, source, constraint=constraint) + assert actual_args == expected_args + assert actual_kwargs == expected_kwargs diff --git a/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py new file mode 100644 index 0000000..e2190a5 --- /dev/null +++ b/lib/ants/tests/io/save/test__check_and_sort_metadata_attributes.py @@ -0,0 +1,140 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. +import logging +from unittest import mock + +import ants.tests.stock as stock +import pytest +from ants.io.save import _check_and_sort_metadata_attributes + + +def test_license_attribute_written(tmp_path): + """Tests that a cube with a license is written out.""" + cube = stock.geodetic(shape=(2, 2)) + license = "This is a cube's license. " + cube.attributes["license"] = license + cube.rename("license test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == license + + +@pytest.mark.filterwarnings( + "ignore:The attribute name licence has been changed to 'license', in line with " + "ANTS working practices.:UserWarning" +) +def test_licence_attribute_written(tmp_path): + """Tests that a cube with a licence is changed and written out.""" + cube = stock.geodetic(shape=(2, 2)) + licence = "This is a cube's license. " + cube.attributes["licence"] = licence + cube.rename("licence test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + assert actual_license == licence + + +def test_loaded_license_written(tmp_path): + """Tests that a cube with a license that resembles the format of a longer license + written in, is written out correctly.""" + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + cube = stock.geodetic(shape=(2, 2)) + cube.attributes["license"] = loaded_license + cube.rename("loaded license test cube") + filename = tmp_path / "test_cube" + _check_and_sort_metadata_attributes([cube], filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.readlines() + assert actual_license == loaded_license + + +def test_log_output(caplog): + """Tests that the logger gives the correct output when a file is written out.""" + cube = stock.geodetic(shape=(2, 2)) + institution = "This data came from University Blah. " + cube.attributes["institution"] = institution + expected_message = ( + "institution has been written to sidecar file filename.institution" + ) + # mocks out the opening of files, so no file is created + with mock.patch("builtins.open"): + with caplog.at_level(logging.INFO): + _check_and_sort_metadata_attributes([cube], "filename") + assert expected_message in caplog.text + + +def test_multiple_cubes(tmp_path): + """Test that multiple_cubes with attributes is written out correctly.""" + # creating three cubes with different license attributes + cube1 = stock.geodetic(shape=(2, 2)) + license1 = "This is a cube's license. " + cube1.attributes["license"] = license1 + cube1.rename("the first cube") + cube2 = stock.geodetic(shape=(2, 2)) + license2 = "This is another cube's license. " + cube2.attributes["license"] = license2 + cube2.rename("the second cube") + cube3 = stock.geodetic(shape=(2, 2)) + license3 = "This is a third cube's license. " + cube3.attributes["license"] = license3 + cube3.rename("the third cube") + cubelist = [cube1, cube2, cube3] + filename = tmp_path / "multiple_cube_test" + # The actual test + _check_and_sort_metadata_attributes(cubelist, filename) + expected_filename = str(filename) + ".license" + with open(expected_filename, "r") as file: + actual_license = file.read() + expected_license = ( + "the first cube = This is a cube's license. \nthe second cube = " + "This is another cube's license. \nthe third cube = This is a third cube's " + "license. \n" + ) + assert actual_license == expected_license + + +def test_all_different_attributes_written_out(): + """Tests that a cube with a multiple different attributes writes + out all metadata files.""" + cube = stock.geodetic(shape=(2, 2)) + attribution = "This data came from an institution. " + cube.attributes["attribution"] = attribution + cube.attributes["restrictions"] = ( + "This data is restricted to be used for testing purposes only." + ) + cube.attributes["license"] = "This is a license for the data" + filename = "test_multiple_attributes" + with mock.patch("ants.io.save._write_metadata_file") as mock_method: + _check_and_sort_metadata_attributes([cube], filename) + + expected_license = mock.call( + ["This is a license for the data"], "test_multiple_attributes", "license" + ) + expected_attribution = mock.call( + ["This data came from an institution. "], + "test_multiple_attributes", + "attribution", + ) + expected_restrictions = mock.call( + ["This data is restricted to be used for testing purposes only."], + "test_multiple_attributes", + "restrictions", + ) + + assert expected_license in mock_method.call_args_list + assert expected_attribution in mock_method.call_args_list + assert expected_restrictions in mock_method.call_args_list diff --git a/lib/ants/tests/io/save/test__check_multiple_attributes.py b/lib/ants/tests/io/save/test__check_multiple_attributes.py new file mode 100644 index 0000000..1d37924 --- /dev/null +++ b/lib/ants/tests/io/save/test__check_multiple_attributes.py @@ -0,0 +1,47 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +from ants.io.save import _check_multiple_attributes + + +def test_one_element_in_list(): + """Tests that when given a list with one element, that is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + +def test_only_one_Cube_in_list(): + """Tests that when given only one cube name, the attributes are returned.""" + expected = [ + "This is an attribution for some of the data. ", + "This is a different attribution for the rest of the data.", + ] + cube_name = ["a cube with multiple attributions"] + actual = _check_multiple_attributes(expected, cube_name) + assert expected == actual + + +def test_all_elements_in_list_same(): + """Tests that when all elements in list are the same, one is returned.""" + expected = ["This is the only attribution given. "] + cube_name = ["a cube with an attribution"] + attribute_list = [ + "This is the only attribution given. ", + "This is the only attribution given. ", + "This is the only attribution given. ", + ] + actual = _check_multiple_attributes(attribute_list, cube_name) + assert expected == actual + + +def test_different_elements_have_cube_names(): + """Tests that when multiple attributes are given, cube names are included.""" + expected = ["cube1 = license 1. \n", "cube2 = license 2. \n"] + licenses = ["license 1. ", "license 2. "] + cube_names = ["cube1", "cube2"] + actual = _check_multiple_attributes(licenses, cube_names) + assert expected == actual diff --git a/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py new file mode 100644 index 0000000..8152be8 --- /dev/null +++ b/lib/ants/tests/utils/cube/test_copy_metadata_attributes.py @@ -0,0 +1,112 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. + +import pytest +from ants.tests import stock +from ants.utils.cube import copy_metadata_attributes + + +def test_adding_new_attribute(): + """Tests that a new attribute will be added to the source cube if it doesn't exist + already.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["license"] = "a test license" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["license"] == "test cube = a test license" + + +def test_modifying_existing_attribute(): + """Tests that an existing attribute will be modified on the source cube.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["references"] = "a test reference" + source.rename("source cube") + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["references"] = "another different test reference" + reference.rename("reference cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + assert ( + source.attributes["references"] + == "source cube = a test reference\n" + + "reference cube = another different test reference" + ) + + +def test_no_reference_attribute(): + """Tests that a source cube attribute with no reference will be left unchanged.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + # Copy the metadata + copy_metadata_attributes(source, reference) + assert source.attributes["restrictions"] == "a test restriction" + + +def test_non_standard_attribute(): + """Tests that an attribute not in the allowed list is not copied over.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["a value not in the allowed list"] = "not in the list" + reference.rename("test cube") + # Copy the metadata + copy_metadata_attributes(source, reference) + expected_msg = "a value not in the allowed list" + with pytest.raises(KeyError, match=expected_msg): + # Check the attribute hasn't been copied over + source.attributes["a value not in the allowed list"] + + +def test_different_attribute_list(): + """Tests that a different attribute list will copy over those attributes.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + # Reference cube: + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["test-attribute"] = "an attribute not in the standard list." + reference.rename("test cube") + copy_metadata_attributes(source, reference, metadata_to_copy=["test-attribute"]) + assert ( + source.attributes["test-attribute"] + == "test cube = an attribute not in the standard list." + ) + + +def test_same_attributes(): + """Tests that if both cubes have the same attribute, nothing happens.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction" + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" + + +def test_different_whitespace(): + """Tests that attributes with different whitespace will be registered as equal.""" + # Source cube: + source = stock.geodetic(shape=(1, 2)) + source.attributes["restrictions"] = "a test restriction" + # Reference cube + reference = stock.geodetic(shape=(3, 4)) + reference.attributes["restrictions"] = "a test restriction " + # Copy the metadata + copy_metadata_attributes(source, reference) + # Check that the attribute is unchanged + assert source.attributes["restrictions"] == "a test restriction" diff --git a/lib/ants/utils/cube.py b/lib/ants/utils/cube.py index 076e919..033af48 100644 --- a/lib/ants/utils/cube.py +++ b/lib/ants/utils/cube.py @@ -1139,6 +1139,65 @@ def inherit_metadata(source, reference): source.attributes["grid_staggering"] = reference.attributes["grid_staggering"] +def copy_metadata_attributes( + source, + reference, + metadata_to_copy=[ + "license", + "attribution", + "restrictions", + "institution", + "acknowledgement", + "references", + ], +): + """ + Inherit cube metadata attributes from a provided reference. + + In-place operation on source cube. + + Metadata attributes from the refrence are added into the source cube. This is done + in the manner of `attribute: cube-name = metadata content` for tracebaility and + allowing for the stacking of metadata. + + Parameters + ---------- + source : :class:`iris.cube.Cube` + Source to have its metadata update. + reference : :class:`iris.cube.Cube` + Reference which defines the metadata to inherit from. + metadata_to_copy : list + A list of metadata attribute keys to copy to the source cube. + """ + + for attribute in metadata_to_copy: + # Only copy an attribute if it exists in the reference cube. + if attribute in reference.attributes: + # If the attribute already exists in the source cube. + if attribute in source.attributes: + # Remove whitepace before comparisons. + source_compare = " ".join(source.attributes[attribute].split()) + reference_compare = " ".join(reference.attributes[attribute].split()) + # If the attributes are the same then nothing needs to be done. + # If the attributes are not the same then both names must be prepended + # to ensure that the metadata is traceable. + if source_compare != reference_compare: + source.attributes[attribute] = ( + source.name() + + " = " + + source.attributes[attribute] + + "\n" + + reference.name() + + " = " + + reference.attributes[attribute] + ) + else: + # If the attribute does not exist in the source cube. + source.attributes[attribute] = ( + reference.name() + " = " + reference.attributes[attribute] + ) + + def set_crs(cube, crs=None): """ Set cube coordinate system. diff --git a/pyproject.toml b/pyproject.toml index a0d1632..76d8a7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ skip = ".git" # Configure pytest. [tool.pytest.ini_options] addopts = "--durations=5" +tmp_path_retention_policy = "none" filterwarnings = [ # During unittesting only, treat all warnings as errors by default: 'error', diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf new file mode 100644 index 0000000..9b79396 --- /dev/null +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-metadata.conf @@ -0,0 +1,15 @@ +[ants_decomposition] +x_split=0 +y_split=0 + +[command] +default=ants-launch ancil_general_regrid.py \ + =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ + =${begin} ${end} --search-method ${search_method} + +[env] +output=${ROSE_DATA}/${ROSE_TASK_NAME} +search_method=kdtree +source=/data/users/theo.geddes/ants-source/n96e_orca_land_cover_fraction +target=${TEST_SOURCES_DIR}/ancil_general_regrid/qrparm.mask.nc +target_type=lsm diff --git a/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf new file mode 100644 index 0000000..e9b1eea --- /dev/null +++ b/rose-stem/app/rose_ana/opt/rose-app-general_regrid_metadata.conf @@ -0,0 +1,3 @@ +[env] +filelist=ancil_general_regrid_metadata + =ancil_general_regrid_metadata.nc diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index 48941eb..3ba6ba1 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -14,6 +14,7 @@ "install_graph" : "install_cold", "ancil_2anc_graph" : "install_cold => ancil_2anc => rose_ana_2anc:fail? => plot_comparisons_2anc", "fill_n_merge_land_cover_graph" : "install_cold => ancil_create_ite_shapefile => ancil_fill_n_merge_land_cover & ancil_fill_n_merge_land_cover_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", + "load_metadata_regrid_graph" : "install_cold => ancil_general_regrid_metadata => rose_ana_general_regrid", "fill_n_merge_invert_mask_graph" : "install_cold => ancil_fill_n_merge_invert_mask & ancil_fill_n_merge_invert_mask_latitude_weighted_kdtree => rose_ana_fill_n_merge:fail? => plot_comparisons_fill_n_merge", "general_regrid_grid_to_grid_graph" : "install_cold => ancil_general_regrid_grid_to_grid => rose_ana_general_regrid:fail? => plot_comparisons_general_regrid install_cold => ancil_general_regrid_grid_to_grid_latitude_weighted_kdtree => rose_ana_general_regrid_latitude_weighted_kdtree:fail? => plot_comparisons_general_regrid_latitude_weighted_kdtree", @@ -254,6 +255,18 @@ fi inherit=ANCIL_GENERAL_REGRID_WITH_TIME_CONSTRAINT script = rose task-run --app-key=ancil_general_regrid_with_time_constraint -O latitude-weighted-kdtree + [[ancil_general_regrid_metadata]] + inherit = ANTS_CORE, ANCIL_GENERAL_REGRID, LARGE + script = rose task-run --app-key=ancil_general_regrid -O metadata + + [[rose_ana_general_regrid_metadata]] + inherit = ROSE_ANA + [[[environment]]] + ROSE_TASK_APP = rose_ana + ROSE_APP_OPT_CONF_KEYS = general_regrid_metadata + ANTS_KGO_DIRECTORY_OVERRIDE = /data/users/theo.geddes/ants-source/KGO/ + + [[rose_ana_general_regrid]] inherit = ROSE_ANA [[[environment]]]