Add the Ability to Load and Save External Metadata

docs/source/tutorial_metadata.rst

@@ -0,0 +1,56 @@
+.. meta::
+   :description lang=en: Tutorial on managing metadata
+   :keywords: metadata, license, licensing, attribution, references, development, tutorial
+   :property=og:locale: en_GB
+
+.. include:: common.txt
+
+
+Managing Metadata
+=================
+
+Metadata in this tutorial refers to information about the data in a file, that does not
+directly affect that data. For example, licensing information or attributions.
+
+Some ancillary file formats cannot include metadata. Metadata can be very important to
+keep alongside the data as often data will have rome form of requirement or restriction
+on it. Because of this, ANTS can handle in external metadata files, provided they match
+the naming convention of `filename.attribute.accepted-metadata`. The current accepted
+metadata attributes are "license", "attribution", "restrictions", "institution",
+"acknowledgement", and "references".
+
+This functionality can be turned off through the use of the argument
+``--ignore-metadata-files`` on the command line or when calling the ANTS load:
+
+.. code-block:: python
+
+    cube = ants.io.load('data', ignore_metadata_files=True)
+
+
+.. note::
+    Any attributes or files referencing licensing should use the 'license' spelling for
+    consistency.
+
+Loading Metadata
+----------------
+
+ANTS can load in external metadata files or 'sidecar' files, if they are kept in the
+same directory as the data files, and have the same name (including the extension).
+Metadata can be loaded alongside all files, including NetCDF files, however ANTS will
+not allow you to load a file with a metadata attribute and a sidecar file with the same
+attribute. This is to prevent unintentional loss or overwriting of metadata information.
+
+.. note::
+    When using wildcards for loading with sidecar files, add the file extension to the
+    end to prevent issues with ANTS attempting to load the sidecar files. E.g.
+    `data*.pp` rather than `data*`.
+
+
+Saving Metadata
+---------------
+
+Metadata will only be saved to a sidecar file, if the ancil loader is used. This is
+because NetCDF files will include the attributes within the file.
+
+The sidecar files produced by ANTS follow the same naming covenstions at those loaded in
+`filename.attribute.accepted-metadata`. Only accepted metadata will be written out.

docs/source/tutorials.rst

@@ -14,3 +14,4 @@ Tutorials
    tutorial_KGO.rst
    tutorial_sources.rst
    tutorial_rose_stem.rst
+   tutorial_metadata.rst

lib/ants/cli/ancil_2anc.py

@@ -37,16 +37,16 @@
 import iris
 
 
-def load_data(source):
+def load_data(source, ignore_metadata_files):
     with warnings.catch_warnings():
         warnings.filterwarnings(
             "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation
         )
-        cubes = ants.io.load.load(source)
+        cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files)
     return cubes
 
 
-def main(source_path, output_path, grid_staggering, netcdf_only):
+def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files):
     """
     Convert specified source file to an ancillary.
 
@@ -66,13 +66,17 @@ def main(source_path, output_path, grid_staggering, netcdf_only):
         written to an ancillary.
 
     """
-    source_cubes = load_data(source_path)
+    source_cubes = load_data(source_path, ignore_metadata_files)
     if grid_staggering is not None:
         for source_cube in source_cubes:
             source_cube.attributes["grid_staggering"] = grid_staggering
 
     if not netcdf_only:
-        save.ancil(source_cubes, output_path)
+        save.ancil(
+            source_cubes,
+            output_path,
+            ignore_writing_metadata_files=ignore_metadata_files,
+        )
     save.netcdf(source_cubes, output_path)
 
     return source_cubes
@@ -98,6 +102,7 @@ def cli_interface():
         args.output,
         args.grid_staggering,
         args.netcdf_only,
+        args.ignore_metadata_files,
     )
 
 

lib/ants/cli/ancil_fill_n_merge.py

@@ -25,6 +25,7 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_files=None,
 ):
     """
     Load the necessary data for performing a merge and fill operation.
@@ -45,6 +46,9 @@ def load_data(
         Datetime to start the processing.
     end: :obj:`datetime`, optional
         Datetime to end the processing.
+    ignore_metadata_files : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
 
 
     Returns
@@ -56,12 +60,16 @@ def load_data(
         respectively.
 
     """
-    primary_cubes = ants.io.load.load(primary_source)
+    primary_cubes = ants.io.load.load(
+        primary_source, ignore_metadata_files=ignore_metadata_files
+    )
     if begin is not None:
         primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end)
     alternate_cubes = None
     if alternate_source:
-        alternate_cubes = ants.io.load.load(alternate_source)
+        alternate_cubes = ants.io.load.load(
+            alternate_source, ignore_metadata_files=ignore_metadata_files
+        )
         if begin is not None:
             alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end)
 
@@ -97,6 +105,7 @@ def main(
     netcdf_only,
     search_method,
     blending_distance,
+    ignore_metadata_files,
 ):
     """
     Perform merge and fill operation on the provided sources.
@@ -152,6 +161,9 @@ def main(
         is applied. Note that this is in units of grid cells, not a physical distance.
         If ``None``, no blending is applied, and there will be a hard edge between
         the two sources.
+    ignore_metadata_files : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
 
     Returns
     -------
@@ -171,6 +183,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_files,
     )
 
     result = primary_cubes
@@ -182,7 +195,7 @@ def main(
         ants.analysis.make_consistent_with_lsm(result, lbm, invert_mask, search_method)
 
     if not netcdf_only:
-        save.ancil(result, output)
+        save.ancil(result, output, ignore_writing_metadata_files=ignore_metadata_files)
     save.netcdf(result, output)
 
     return result
@@ -271,6 +284,7 @@ def cli_interface():
         netcdf_only=args.netcdf_only,
         search_method=args.search_method,
         blending_distance=args.blending_distance,
+        ignore_metadata_files=args.ignore_metadata_files,
     )
 
 

lib/ants/cli/ancil_general_regrid.py

@@ -42,12 +42,17 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_files=None,
 ):
-    source_cubes = ants.io.load.load(source)
+    source_cubes = ants.io.load.load(
+        source, ignore_metadata_files=ignore_metadata_files
+    )
     if begin is not None:
         source_cubes = create_time_constrained_cubes(source_cubes, begin, end)
     if target_grid:
-        target_cube = ants.io.load.load_grid(target_grid)
+        target_cube = ants.io.load.load_grid(
+            target_grid, ignore_metadata_files=ignore_metadata_files
+        )
     else:
         target_cube = ants.io.load.load_landsea_mask(
             target_landseamask, land_fraction_threshold
@@ -76,6 +81,7 @@ def main(
     save_ukca,
     netcdf_only,
     search_method,
+    ignore_metadata_files,
 ):
     """
     General regrid application top level call function.
@@ -118,6 +124,9 @@ def main(
         provided source(s) consistent with the provided land sea mask.
         This should only be provided if a target land sea mask is also
         provided via target_lsm_path.
+    ignore_metadata_files : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
 
     Returns
     -------
@@ -132,6 +141,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_files,
     )
     if ants.utils.cube._is_ugrid(target_cube):
         raise ValueError(
@@ -149,7 +159,11 @@ def main(
         save.ukca_netcdf(regridded_cubes, output_path)
     else:
         if not netcdf_only:
-            save.ancil(regridded_cubes, output_path)
+            save.ancil(
+                regridded_cubes,
+                output_path,
+                ignore_writing_metadata_files=ignore_metadata_files,
+            )
         save.netcdf(regridded_cubes, output_path)
 
     return regridded_cubes
@@ -210,6 +224,7 @@ def cli_interface():
         args.save_ukca,
         args.netcdf_only,
         args.search_method,
+        args.ignore_metadata_files,
     )
 
 

lib/ants/command_parse.py

@@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False):
             sources <SOURCE1> <SOURCE2> ... <SOURCEN>     Source data path(s).
             --output <OUTPUT>, -o <OUTPUT>                Output filepath
             --ants-config <ANTS_CONFIG>                   Configuration path.
+            --ignore-metadata-files                       Stops the application from
+                                                          searching for metadata files
+                                                          with the same name as the
+                                                          source and loading them
+                                                          alongside the source.
 
         Additionally, there are standardised optional arguments and these are
         activated by passing the relevant keyword argument.  See 'Parameters'
@@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False):
             help="Only write out a netCDF file.",
             required=False,
         )
+        self.add_argument(
+            "--ignore-metadata-files",
+            action="store_true",
+            help="Stops the application from searching for metadata files with "
+            "the same name as the source and loading them alongside the source.",
+            required=False,
+        )
         if time_constraints:
             self.add_argument(
                 "--begin",