From 6a2e88eb374fae566b55bb4d500f3df170b0948e Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Mon, 11 May 2026 15:01:40 +0100 Subject: [PATCH 01/47] Refactor wet model drivers and controllers Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/common.py | 96 +- Coupled_Drivers/dr_env_lib/nemo_def.py | 12 +- Coupled_Drivers/dr_env_lib/ocn_cont_def.py | 16 +- Coupled_Drivers/error.py | 35 +- Coupled_Drivers/fcm_make/driver.cfg | 4 +- Coupled_Drivers/link_drivers | 42 +- Coupled_Drivers/nemo_driver.py | 1124 ++++------------- Coupled_Drivers/run_model | 61 +- Coupled_Drivers/unittests/test.py | 25 +- .../test_aprun_command_construction.py | 87 +- Coupled_Drivers/unittests/test_envar_lib.py | 11 +- 11 files changed, 359 insertions(+), 1154 deletions(-) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index d3470cfd..f1e397bb 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2022-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,11 +18,9 @@ Common functions and classes required by multiple model drivers ''' - - -import datetime -import glob -import shutil +#The from __future__ imports ensure compatibility between python2.7 and 3.x +from __future__ import absolute_import +import copy import re import os import sys @@ -32,7 +30,6 @@ import error import inc_days - class ModNamelist(object): ''' Modify a fortran namelist. This will not add any new variables, only @@ -49,7 +46,7 @@ def __init__(self, filename): def var_val(self, variable, value): ''' Create a container of variable name, value pairs to be updated. Note - that if a variable doesn't exist in the namelist file, then it + that if a variable doesn't exisit in the namelist file, then it will be ignored ''' if isinstance(value, str): @@ -83,7 +80,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): ''' Find the work directory for the previous cycle. Takes as argument the current cyclepoint, the path to the current work directory, and - the current taskname, a value specifying multiple tasks within + the current taskname, a value specifying multiple tasks within same cycle (e.g. coupled_run1, coupled_run2) as used in coupled NWP and returns an absolute path. ''' @@ -100,7 +97,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): return previous_workdir - else: + else: cyclesdir = os.sep.join(workdir.split(os.sep)[:-2]) #find the work directory for the previous cycle work_cycles = os.listdir(cyclesdir) @@ -121,7 +118,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): sys.stderr.write('[FAIL] Can not find previous work directory for' ' task %s\n' % taskname) sys.exit(error.MISSING_DRIVER_FILE_ERROR) - + return os.path.join(cyclesdir, previous_task_cycle, taskname) @@ -139,6 +136,7 @@ def get_filepaths(directory): return file_paths + def open_text_file(name, mode): ''' Provide a common function to open a file and provide a suitiable error @@ -152,7 +150,7 @@ def open_text_file(name, mode): 'a+':'updating (appending)'} if mode not in list(modes.keys()): options = '' - for k in modes: + for k in modes.keys(): options += ' %s: %s\n' % (k, modes[k]) sys.stderr.write('[FAIL] Attempting to open file %s, do not recognise' ' mode %s. Please use one of the following modes:\n%s' @@ -188,6 +186,28 @@ def remove_file(filename): else: return False +def remove_trailing_slash(path_str): + ''' + Takes a string of a path and removes any trailing slashes and spaces if + there are any present + ''' + trailing_slash_regex = r'[\w/]*\w+([\s/]+)$' + match = re.match(trailing_slash_regex, path_str) + if match: + num_trailing_slashes = len(match.group(1)) + return path_str[:-num_trailing_slashes] + return path_str + + +def remove_trailing_slash_orig(path_str): + ''' + Takes a string of a path and removes the trailing slash if there is one + present + ''' + if path_str[-1] == '/': + return path_str[:-1] + return path_str + def setup_runtime(common_env): ''' Set up model run length in seconds based on the model suite @@ -324,53 +344,3 @@ def set_aprun_options(nproc, nodes, ompthr, hyperthreads, ss): rose_launcher_preopts = "-ss " + rose_launcher_preopts return rose_launcher_preopts - - -def _sort_hist_dirs_by_date(dir_list): - ''' - Sort a list of history directories by date - ''' - # Pattern that defines the name of the history directories, - # which contain a date of the form YYYYmmddHHMM. - pattern = r'\.(\d{12})' - - try: - dir_list.sort(key=lambda dname: datetime.datetime.strptime( - re.search(pattern, dname).group(1), '%Y%m%d%H%M')) - except AttributeError: - msg = '[FAIL] Cannot order directories: %s' % " ".join(dir_list) - sys.stderr.write(msg) - sys.exit(error.IOERROR) - - return dir_list - - -def remove_latest_hist_dir(old_hist_dir): - ''' - If a model task has failed, then removed the last created history - directory, before a new one is created, associated with the - re-attempt. - ''' - # Replace the regex pattern that defines the history directory - # name (that contains a date of the format YYYYmmddHHMM) with a - # generic pattern so that we can perform the directory glob. - history_pattern = re.sub( - r'\.\d{12}', '.????????????', old_hist_dir) - - # Find and sort the history directories, and delete - # the latest one, corresponding to the last entry in - # the list. - history_dirs = glob.glob(history_pattern) - history_dirs = _sort_hist_dirs_by_date(history_dirs) - - msg = '[INFO] Found history directories: %s \n' % ' '.join( - history_dirs) - sys.stdout.write(msg) - - latest_hist_dir = history_dirs[-1] - msg = ("[WARN] Re-attempting failed model step. \n" - "[WARN] Clearing out latest history \n" - "[WARN] directory %s. \n" % latest_hist_dir) - sys.stdout.write(msg) - - shutil.rmtree(latest_hist_dir) diff --git a/Coupled_Drivers/dr_env_lib/nemo_def.py b/Coupled_Drivers/dr_env_lib/nemo_def.py index 98e78b2a..0f1cf04f 100644 --- a/Coupled_Drivers/dr_env_lib/nemo_def.py +++ b/Coupled_Drivers/dr_env_lib/nemo_def.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -22,18 +22,20 @@ # Variables required for both initalise and finalise NEMO_ENVIRONMENT_VARS_COMMON = { 'NEMO_NL': {'default_val': 'namelist_cfg'}, - 'L_OCN_PASS_TRC': {'default_val': 'false'} + 'L_OCN_PASS_TRC': {'default_val': 'false'}, + 'NEMO_NPROC': {'desc': 'Number of NEMO processors'}, + 'RST_LINK_DIR': {'default_val': '.', + 'desc': 'Subdirectory of $CYCL_TASK_WORK_DIR containing' \ + ' ocean restart links'} } NEMO_ENVIRONMENT_VARS_INITIAL = { 'OCEAN_EXEC': {'desc': ('Ocean executable (OCEAN_EXEC=)')}, - 'NEMO_NPROC': {'desc': 'Number of NEMO processors'}, 'NEMO_IPROC': {'desc': 'Number of NEMO processors in the i direction'}, 'NEMO_JPROC': {'desc': 'Number of NEMO processors in the j direction'}, 'NEMO_VERSION': {}, 'OCEAN_LINK': {'default_val': 'ocean.exe'}, - 'OCN_RES': {'default_val': ''}, 'NEMO_NL': {'default_val': 'namelist_cfg'}, 'NEMO_START': {'default_val': ''}, 'NEMO_ICEBERGS_START': {'default_val': ''}, diff --git a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py index 6acf0563..e087f0b0 100644 --- a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py +++ b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,17 +18,17 @@ Definition of the environment variables required for the ocean controllers ''' +SI3_ENVIRONMENT_VARS_COMMON = {'SI3_NL': {'default_val': 'namelist_ice_cfg'}} +SI3_ENVIRONMENT_VARS_INITIAL = {'SI3_START': {'default_val': ''}} -SI3_ENVIRONMENT_VARS_INITIAL = { - 'SI3_START': {'default_val': ''}, - 'SI3_NL': {'default_val': 'namelist_ice_cfg'}} +SI3_ENVIRONMENT_VARS_INITIAL = {**SI3_ENVIRONMENT_VARS_COMMON, + **SI3_ENVIRONMENT_VARS_INITIAL} +SI3_ENVIRONMENT_VARS_FINAL = SI3_ENVIRONMENT_VARS_COMMON TOP_ENVIRONMENT_VARS_COMMON = { 'TOP_NL': {'default_val': 'namelist_top_cfg'}} TOP_ENVIRONMENT_VARS_INITIAL = { - 'TOP_START': {'default_val': ''}, - 'CONTINUE': {'default_val': 'false'}, - 'CONTINUE_FROM_FAIL': {'default_val': 'false'}} + 'TOP_START': {'default_val': ''}} TOP_ENVIRONMENT_VARS_INITIAL = {**TOP_ENVIRONMENT_VARS_COMMON, **TOP_ENVIRONMENT_VARS_INITIAL} diff --git a/Coupled_Drivers/error.py b/Coupled_Drivers/error.py index 5e4d69a3..3b23466d 100644 --- a/Coupled_Drivers/error.py +++ b/Coupled_Drivers/error.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -23,9 +23,6 @@ # Python errors VERSION_ERROR = 1 -# Import Error -IMPORT_ERROR = 2 - # File I/O errors IOERROR = 100 @@ -53,6 +50,9 @@ # Invalid argument to driver script INVALID_DRIVER_ARG_ERROR = 202 +# Missing file contents +MISSING_FILE_CONTENTS_ERROR = 203 + # Missing file required by a controller MISSING_CONTROLLER_FILE_ERROR = 251 @@ -139,6 +139,9 @@ # Unable to create remapping file for mapping type MISSING_RMP_MAPPING = 816 +# Size of 1D grid is unknown +UNKNOWN_1D_GRID_SIZE = 817 + # Ocean resolution for Snr<->Jnr coupling is not specified NO_OCN_RESOL = 818 @@ -181,17 +184,11 @@ # Not on ORCA grid NOT_ORCA_GRID = 833 -# Not declared ocean resolution when creating namcouple on fly after NEMO3.6 -NOT_DECLARE_OCN_RES = 834 - -# Unknown ocean resolution -UNKNOWN_OCN_RESOL = 835 - # Missing namelist oasis_ocn_send_nml from OASIS_OCN_SEND -MISSING_OASIS_OCN_SEND_NML = 836 +MISSING_OASIS_OCN_SEND_NML = 834 # Missing entry oasis_ocn_send from namelist oasis_ocn_send_nml -MISSING_OASIS_OCN_SEND = 837 +MISSING_OASIS_OCN_SEND = 835 # Missing OASIS send file for UM MISSING_OASIS_ATM_SEND = 840 @@ -257,15 +254,3 @@ # Not found the coupling namelist in namelist_cfg MISSING_NAMSBC_CPL = 871 - -# Missing the JULES river resolution namelist -MISSING_RIVER_RESOL_NML = 872 - -# Missing the JULES river resolution parameters -MISSING_RIVER_RESOL = 873 - -# Missing OASIS send file for JULES river -MISSING_OASIS_RIV_SEND = 874 - -# Missing namelist oasis_riv_send_nml from OASIS_RIV_SEND -MISSING_OASIS_RIV_SEND_NML = 875 diff --git a/Coupled_Drivers/fcm_make/driver.cfg b/Coupled_Drivers/fcm_make/driver.cfg index 2f63363b..05f3d7a4 100644 --- a/Coupled_Drivers/fcm_make/driver.cfg +++ b/Coupled_Drivers/fcm_make/driver.cfg @@ -14,8 +14,8 @@ extract.ns = drivers extract.location{primary}[drivers] = fcm:moci.xm extract.location[drivers] = $driver_base${driver_rev} extract.path-excl[drivers] = / -extract.path-incl[drivers] = Coupled_Drivers Utilities/NGMS_utils/ngms_namcouple_gen Utilities/NGMS_utils/ngms_suite_lib +extract.path-incl[drivers] = Coupled_Drivers -$install_common= +$install_common = cice_driver.py common.py write_namcouple.py default_couplings.py error.py inc_days.py jnr_driver.py link_drivers mct_driver.py nemo_driver.py OASIS_fields run_model save_um_state.py time2days.py um_driver.py update_namcouple.py update_nemo_nl.py xios_driver.py plot_cpmip_summary.py cpmip_controller.py mct_validate.py ocn_lib.py nemo_runtime_namcouple.py nemo_driver.py build.target = $install_common diff --git a/Coupled_Drivers/link_drivers b/Coupled_Drivers/link_drivers index e5cb3439..d7dd6be2 100755 --- a/Coupled_Drivers/link_drivers +++ b/Coupled_Drivers/link_drivers @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -19,19 +19,24 @@ DESCRIPTION to run a given configuration ''' - +#The from __future__ imports ensure compatibility between python2.7 and 3.x +from __future__ import absolute_import import importlib import sys import os import argparse import error import common +import write_namcouple import dr_env_lib.common_def import dr_env_lib.env_lib #A tuple for for the earlist version of python required to run the drivers REQUIRED_VERSION = (3, 6) +# A submodel is a model which runs within another model, and doesnt require its # own driver or controller, for example si3 is a submodel of NEMO +SUBMODELS = ['si3', 'top'] + def _check_drivers(models): ''' Takes in our space separated list of models, and ensure that a driver is @@ -39,8 +44,8 @@ def _check_drivers(models): ''' sys.stdout.write('MODELS %s\n' % models) - models = models.split(' ') - for model in models[:]: + models = [model for model in models.split(' ') if model not in SUBMODELS] + for model in models: drivername = '%s_driver.py' % model if not os.path.isfile(drivername): # check to see if this is a controller rather than a driver @@ -116,23 +121,17 @@ def _setup_common(): return common_env -def _export_cmd_str(launchcmds, common_env): +def _export_cmd_str_XC40(launchcmds, common_env): ''' - Create an MPI launch command for the model configuration + Create an aprun launch command for the coupled configuration on the + XC40 ''' - message_str = '[LINK_DRIVERS] attemting to run with command: %s\n' - # If a custom command is used we write that straight into cmd variable - # rather than constructing the command string - if common_env['CUSTOM_LAUNCH_COMMAND']: - cmd = common_env['CUSTOM_LAUNCH_COMMAND'] - sys.stdout.write(message_str % cmd) - return cmd - cmd = '%s ' % common_env['ROSE_LAUNCHER'] for launchcmd in launchcmds: if launchcmd != '': cmd += launchcmd+' : ' - sys.stdout.write(message_str % cmd[:-3]) + sys.stdout.write('[LINK_DRIVERS] attemting to run with command: %s\n' + % cmd[:-3]) return cmd[:-3] @@ -141,9 +140,9 @@ def _export_cmd_str(launchcmds, common_env): if __name__ == '__main__': - # Owing to the limitations of the MPI launchers and passing information - # between the shell and the python drivers, prepare two command files, one - # containing a command to execute, the other containing a command allowing + # Owing to the limitations of aprun and passing information between the + # shell and the python drivers, prepare two command files, one containing + # an aprun command to execute, the other containing a command allowing # environment variables to be exported try: @@ -203,7 +202,6 @@ if __name__ == '__main__': # Create the namcouple file if not run_info['l_namcouple']: - import write_namcouple write_namcouple.write_namcouple( common_env, run_info, coupling_list) @@ -211,8 +209,8 @@ if __name__ == '__main__': # environment variable definitions that can be run by the shell envar_str = dr_env_lib.env_lib.string_for_export(envinsts) - # Create the MPI launcher command - cmd_str = _export_cmd_str(launchcmds, common_env) + # Create the aprun command + cmd_str = _export_cmd_str_XC40(launchcmds, common_env) # Write the export commands for common and model specific environment # variables (envar_str) into the file driver_envar diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index e8167725..d5603ff2 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,801 +18,280 @@ Driver for the NEMO 3.6 model, called from link_drivers. Note that this does not cater for any earlier versions of NEMO ''' -import re +import collections +import importlib import os -import time -import datetime -import sys -import glob +import re import shutil -import inc_days +import sys + import common import error - -try: - import cf_units -except ImportError: - IMPORT_ERROR_MSG = ('Unable to import cf_units. Ensure the scitools module' - 'has been loaded first.') - sys.exit(IMPORT_ERROR_MSG) - -import dr_env_lib.nemo_def -import dr_env_lib.env_lib -try: - import f90nml -except ImportError: - pass - -# Here, "top" refers to the NEMO TOP passive tracer system. It does not -# imply anything to do with being in overall control or at the head of -# any form of control heirarchy. -import top_controller - -import si3_controller +import nemo_lib +import nemo_restart_lib +import nemo_runtime_namcouple +import ocn_lib +import update_nemo_nl # Define errors for the NEMO driver only SERIAL_MODE_ERROR = 99 -# Ocean resolutions -OCEAN_RESOLS = {'orca2': [182, 149], - 'orca1': [362, 332], - 'orca025': [1442, 1021], - 'orca12': [4322, 3059], - 'orca36': [12960, 10850]} -def _check_nemonl(envar_container): +def _nemo_driver_assertions(nemo_envar): ''' - As the environment variable NEMO_NL is required by both the setup - and finalise functions, this will be encapsulated here + Assertions prior to the running of the drivers, to ensure that we are + running in a parallel configuration, and the NEMO version is 3.6 or later ''' - # Information will be retrieved from this file during the running of the - # driver, so check it exists - if not os.path.isfile(envar_container['NEMO_NL']): - sys.stderr.write('[FAIL] Can not find the nemo namelist file %s\n' % - envar_container['NEMO_NL']) - sys.exit(error.MISSING_DRIVER_FILE_ERROR) - else: - return 0 + assert int(nemo_envar['NEMO_VERSION']) >= 306, \ + '[FAIL] The python drivers are only valid for NEMO versions' \ + ' 3.6 and later.\n' -def _get_nemorst(nemo_nl_file): - ''' - Retrieve the nemo restart directory from the nemo namelist file - ''' - ocerst_rcode, ocerst_val = common.exec_subproc([ \ - 'grep', 'cn_ocerst_outdir', nemo_nl_file]) - if ocerst_rcode == 0: - nemo_rst = re.findall(r'[\"\'](.*?)[\"\']', ocerst_val)[0] - if nemo_rst[-1] == '/': - nemo_rst = nemo_rst[:-1] - return nemo_rst - return None - -def _get_ln_icebergs(nemo_nl_file): - ''' - Interrogate the nemo namelist to see if we are running with icebergs, - Returns boolean, True if icebergs are used, False if not - ''' - icb_rcode, icb_val = common.exec_subproc([ \ - 'grep', 'ln_icebergs', nemo_nl_file]) - if icb_rcode != 0: - sys.stderr.write('Unable to read ln_icebergs in &namberg namelist' - ' in the NEMO namelist file %s\n' - % nemo_nl_file) - sys.exit(error.SUBPROC_ERROR) - else: - if 'true' in icb_val.lower(): - return True - return False + assert int(nemo_envar['NEMO_NPROC']) > 1, \ + '[FAIL] Nemo driver does not support the running of NEMO' \ + ' in serial mode\n' -def _verify_nemo_rst(cyclepointstr, nemo_rst, nemo_nl, nemo_nproc, nemo_version): +def _run_submodel_custom(common_env, nemo_envar, ln_restart, restart_ctl): ''' - Verify that the full set of nemo restart files match. Currently this - is limited to the icebergs restart file. We require either a single - restart file, or a number of restart files equal to the number of - nemo processors. + If the submodels have unique requirements we run those here ''' - restart_files = [f for f in os.listdir(nemo_rst) if - 'restart' in f] - - - if _get_ln_icebergs(nemo_nl): - - if nemo_version < 402: - # Pre nemo 4.2 compatibility - nemo_icb_regex = r'_icebergs_%s_restart(_\d+)?\.nc' % cyclepointstr + if 'top' in common_env['models']: + # ln_trcdta appears to always be the opposite of ln_restart, so we + # set it on that basis, though if this is correct it would seem to + # be redundant which should be addressed in the NEMO base code. + # These settings are based purely on the logic employed by the + # forerunner to this code in the MEDUSA-adapted UM10.6 GC3-coupled + # control script. + if ln_restart == ".true.": + ln_trcdta = ".false." + elif ln_restart == ".false.": + ln_trcdta = ".true." else: - # Post nemo 4.2 compatibility - nemo_icb_regex = r'_%s_restart_icb(_\d+)?\.nc' % cyclepointstr - - icb_restart_files = [f for f in restart_files if - re.findall(nemo_icb_regex, f)] - - # we can have a single rebuilt file, number of files equal to - # number of nemo processors, or rebuilt file and processor files. - if len(icb_restart_files) not in (1, nemo_nproc, nemo_nproc+1): - sys.stderr.write('[FAIL] Unable to find iceberg restart files for' - ' this cycle. Must either have one rebuilt file,' - ' as many as there are nemo processors (%i) or' - ' both rebuilt and processor files.' - '[FAIL] Found %i iceberg restart files\n' - % (nemo_nproc, len(icb_restart_files))) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - - -def _calc_current_model_date(model_basis_time, time_step, num_steps, - calendar): + sys.stderr.write('[FAIL] TOP: invalid ln_restart value:' + ' %s\n' % ln_restart) + sys.exit(error.INVALID_LOCAL_ERROR) + # Update the TOP namelist + update_nemo_nl.update_top_nl(nemo_envar, ln_restart, restart_ctl, + ln_trcdta) + + if 'si3' in common_env['models']: + update_nemo_nl.update_si3_nl(nemo_envar) + +def _run_submodel_controllers(nemo_envar, common_env, restart_ctl, + nemo_dump_time, controller_mode): ''' - Calculate the current model date using the basis time, - and the number of time-steps covered in a given model run. - - :arg: datetime model_basis_time : Model basis time - :arg: int time_step : Ocean model timestep (in seconds) - :arg: int num_steps : Num. timesteps covered in this - model run - :arg: string calendar : Calendar used in the model run - + Import and run the TOP/SI3 controllers if required ''' - ref_date_format = 'seconds since %Y-%m-%d %H:%M:%S' - - # modify the calendar names for compatability with cf_units module - if calendar == "360day": - calendar = "360_day" - if calendar == "365day": - calendar = "365_day" - - # Provide a reference time for the timestep incrementation. - ref_time = model_basis_time.strftime(ref_date_format) - - model_progress_secs = cf_units.date2num( - model_basis_time, ref_time, calendar=calendar) + (time_step * num_steps) - - current_model_date = cf_units.num2date( - model_progress_secs, ref_time, calendar=calendar) - - return current_model_date - - -def _verify_fix_rst(restartdate, nemo_rst, model_basis_time, time_step, - num_steps, calendar): - ''' - Verify that the restart file for nemo corresponds to the model - time reached within a given model run. If they don't match, then - make sure that nemo restarts from the correct restart date - - :arg: string restartdate : NEMO dump file date - :arg: string nemo_rst : Path to NEMO restart files - :arg: string model_basis_time : Model basis time - :arg: int time_step : Ocean time-step (in seconds) - :arg: int num_steps : Num. of time-steps covered - :arg: string calendar : Calendar used in model - - ''' - # Calculate the model restart time based on the start date of the - # last calculated model step, the time-step and the number of - # steps. Then convert the date format. - model_basis_datetime = datetime.datetime.strptime( - model_basis_time, "%Y%m%d") - - model_restart_date = _calc_current_model_date( - model_basis_datetime, time_step, num_steps, calendar) - - model_restart_date = model_restart_date.strftime('%Y%m%d') - - if restartdate == model_restart_date: - sys.stdout.write('[INFO] Validated NEMO restart date\n') + submodels = [] + # We check for the presence of lower case t, as in a lowered (TRUE, True, + # or true) + if 't' in nemo_envar['L_OCN_PASS_TRC'].lower(): + submodels.append('top') + sys.stdout.write('[INFO] nemo_driver: Passive tracer code is ' + 'active.\n') else: - # Write the message to both standard out and standard error - msg = '[WARN] The NEMO restart data does not match the ' \ - ' current model time\n.' \ - ' Current model date is %s\n' \ - ' NEMO restart time is %s\n' \ - '[WARN] Automatically removing NEMO dumps ahead of ' \ - 'the current model date, and pick up the dump at ' \ - 'this time\n' % (model_restart_date, restartdate) - sys.stdout.write(msg) - sys.stderr.write(msg) - #Remove all nemo restart files that are later than the correct - #cycle times - #Make our generic restart regular expression, to cover normal NEMO - #restart, and potential iceberg, SI3 or passive tracer restart files, - #for both the rebuilt and non rebuilt cases - generic_rst_regex = r'(icebergs)?.*restart(_trc)?(_ice)?(_icb)?(_\d+)?\.nc' - all_restart_files = [f for f in os.listdir(nemo_rst) if - re.findall(generic_rst_regex, f)] - for restart_file in all_restart_files: - fname_date = re.findall(r'\d{8}', restart_file)[0] - if fname_date > model_restart_date: - common.remove_file(os.path.join(nemo_rst, restart_file)) - restartdate = model_restart_date - return restartdate - + sys.stdout.write('[INFO] nemo_driver: ' + 'Passive tracer code not active.\n') + for submodel in submodels: + controller_name = '%s_controller' % submodel + sys.stdout.write('[INFO] Calling %s in %s mode\n' % + (controller_name, controller_mode)) + controller_mod = importlib.import_module(controller_name) + controller_mod.run_controller(common_env, + restart_ctl, + int(nemo_envar['NEMO_NPROC']), + common_env['RUNID'], + common_env['DRIVERS_VERIFY_RST'], + nemo_dump_time, + controller_mode) -def _load_environment_variables(nemo_envar): +def _processor_restart_files_nrun(nemo_envar, common_env, nemo_init_dir): ''' - Load the NEMO environment variables required for the model run into the - nemo_envar container + Set up links to processor restart files for NRUN and return ln_restart ''' - # Load the nemo namelist environment variable - nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( - nemo_envar, dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL) - _ = _check_nemonl(nemo_envar) - - return nemo_envar - - -def _setup_dates(common_env): + nemo_warn = 'NEMO_START not set\nNEMO will use climatology\n' + ln_restart = ocn_lib.setup_nrun(nemo_envar['NEMO_START'], + nemo_envar['RST_LINK_DIR'], + 'restart', nemo_init_dir, + nemo_warn) + + # Set up Icebergs NRUN + icebergs_warn = 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n' + _ = ocn_lib.setup_nrun(nemo_envar['NEMO_ICEBERGS_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_icebergs', nemo_init_dir, + icebergs_warn) + + # Set up SI3 NRUN + if 'si3' in common_env['models']: + si3_warn = 'New SI3 run\n' + _ = ocn_lib.setup_nrun(nemo_envar['SI3_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_ice', nemo_init_dir, si3_warn) + + # Set up TOP NRUN + if 'top' in common_env['models']: + top_warn = 'New TOP run\n' + _ = ocn_lib.setup_nrun(nemo_envar['TOP_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_trc', nemo_init_dir, top_warn) + return ln_restart + +def _processor_restart_files_crun(nemo_init_dir, restart_link_dir, + nemo_nproc, nemo_dump_time, runid): ''' - Setup the dates for the NEMO model run + Compile a list of relevant restart files for CRUNs ''' - calendar = common_env['CALENDAR'] - - sys.stdout.write('[INFO] NEMO calendar= %s ' % calendar) - if calendar == '360day': - calendar = '360' - nleapy = 30 - elif calendar == '365day': - calendar = '365' - nleapy = 0 - elif calendar == 'gregorian': - nleapy = 1 + Restarts = collections.namedtuple( + 'Restarts', 'model file_label rst_label') + # Create a list of the various Restarts + restart_types = [ + Restarts('NEMO', '%so_%s_restart', 'restart'), + Restarts('SI3', '%so_%s_restart_ice', 'restart_ice'), + Restarts('Icebergs', '%so_icebergs_%s_restart', 'restart_icebergs'), + Restarts('TOP', '%so_%s_restart_trc', 'restart_trc')] + for i_restart in restart_types: + # Create links for restart files - either rebuilt or processor restarts + file_label = i_restart.file_label % (runid, nemo_dump_time) + ocn_lib.setup_restart(nemo_init_dir, restart_link_dir, + file_label, i_restart.rst_label, nemo_nproc, + i_restart.model) + +def _crun_timesteps(nemo_rst_date_bool, nemo_dump_time, nemo_step_int, + nemo_last_step, common_env): + '''Set up the timesteps for a CRUN''' + if not nemo_rst_date_bool: + # Nemo dump time is relative to the start of a model run and is an + # integer + nemo_dump_time = int(nemo_dump_time) + completed_days = nemo_dump_time * (nemo_step_int / 86400) + sys.stdout.write('[INFO] Nemo has previously completed %i days\n' % + completed_days) + ln_restart = ".true." + restart_ctl = 2 + if common_env['CONTINUE_FROM_FAIL'] == 'true': + # This is only used for coupled NWP where we don't have dates in + # NEMO restart file names + nemo_next_step = int(nemo_dump_time)+1 else: - sys.stderr.write('[FAIL] Calendar type %s not recognised\n' % - calendar) - sys.exit(error.INVALID_EVAR_ERROR) + nemo_next_step = nemo_last_step + 1 + return ln_restart, restart_ctl, nemo_next_step - #turn our times into lists of integers - model_basis = [int(i) for i in common_env['MODELBASIS'].split(',')] - run_start = [int(i) for i in common_env['TASKSTART'].split(',')] - run_length = [int(i) for i in common_env['TASKLENGTH'].split(',')] - run_days = inc_days.inc_days(run_start[0], run_start[1], run_start[2], - run_length[0], run_length[1], run_length[2], - calendar) - return nleapy, model_basis, run_start, run_length, run_days +def _nrun_timesteps(nemo_first_step): + '''Set up the timesteps for an NRUN''' + restart_ctl = 0 + nemo_next_step = nemo_first_step + nemo_last_step = nemo_first_step - 1 + return restart_ctl, nemo_next_step, nemo_last_step +def _verify_restart(common_env, nemo_envar, nemo_dump_time, nemo_rst): + ''' + Verify the dump time against cycle time if appropriate, do the + automatic fix, and check all other restart files match + ''' + if common_env['DRIVERS_VERIFY_RST'] == 'True': + nemo_dump_time = nemo_restart_lib.verify_fix_rst( + nemo_dump_time, + common_env['CYLC_TASK_CYCLE_POINT'], nemo_rst) + return nemo_dump_time def _setup_executable(common_env): ''' - Setup the environment and any files required by the executable + Set up the environment and any files required by the executable ''' - # Create the environment variable container - nemo_envar = dr_env_lib.env_lib.LoadEnvar() - # Load the environment variables required - nemo_envar = _load_environment_variables(nemo_envar) + # Load the environment variables required + nemo_envar, models = nemo_lib.load_environment_variables( + 'setup', common_env['models']) + common_env['models'] = models #Link the ocean executable common.remove_file(nemo_envar['OCEAN_LINK']) os.symlink(nemo_envar['OCEAN_EXEC'], nemo_envar['OCEAN_LINK']) - # Setup date variables + # Set up date variables nleapy, model_basis, run_start, \ - run_length, run_days = _setup_dates(common_env) - - # NEMO model setup - if int(nemo_envar['NEMO_VERSION']) < 306: - sys.stderr.write('[FAIL] The python drivers are only valid for nemo' - ' versions greater than 3.6') - sys.exit(error.INVALID_COMPONENT_VER_ERROR) - - # Read restart from nemo namelist - restart_direcs = [] - nemo_rst = _get_nemorst(nemo_envar['NEMO_NL']) - if nemo_rst: - restart_direcs.append(nemo_rst) - icerst_rcode, icerst_val = common.exec_subproc([ \ - 'grep', 'cn_icerst_dir', nemo_envar['NEMO_NL']]) - if icerst_rcode == 0: - ice_rst = re.findall(r'[\"\'](.*?)[\"\']', icerst_val)[0] - if ice_rst[-1] == '/': - ice_rst = ice_rst[:-1] - restart_direcs.append(ice_rst) - - for direc in restart_direcs: - # Strip white space - direc = direc.strip() - - # Check for trailing slashes in directory names and strip them - # out if they're present. - if direc.endswith('/'): - direc = direc.rstrip('/') + run_length, run_days = nemo_lib.setup_dates(common_env) - if os.path.isdir(direc) and (direc not in ('./', '.')) and \ - common_env['CONTINUE'] == 'false': - sys.stdout.write('[INFO] directory is %s\n' % direc) - sys.stdout.write('[INFO] This is a New Run. Renaming old NEMO' - ' history directory\n') + # Make the required assertions + _nemo_driver_assertions(nemo_envar) - # In seasonal forecasting, we automatically apply - # short-stepping to re-try the model. Before re-attempting - # it, remove the associated NEMO history directory. - old_hist_dir = "%s.%s" % (direc, time.strftime("%Y%m%d%H%M")) + # Read restart from current nemo namelists + nemo_rst, is_icebergs, ice_rst, top_rst = nemo_lib.read_current_cycle_nl( + common_env, nemo_envar) - if (common_env['SEASONAL'] == 'True' and - int(common_env['CYLC_TASK_TRY_NUMBER']) > 1): - common.remove_latest_hist_dir(old_hist_dir) + # we dont want any duplicates + restart_direcs = list( + collections.OrderedDict.fromkeys( + [direc for direc in [nemo_rst, ice_rst, top_rst] if direc])) - os.rename(direc, old_hist_dir) - os.makedirs(direc) - elif not os.path.isdir(direc): - sys.stdout.write('[INFO] Creating NEMO restart directory:\n %s' % - direc) - os.makedirs(direc) + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) - # Compile a list of NEMO, seaice and iceberg restart files, if any exist. - # We look for files conforming to the naming convention: - # _yyyymmdd_restart_.nc OR - # _yyyymmdd_restart_icb_.nc where - # may itself contain underscores, hence we - # do not parse details based on counting the number of underscores. - nemo_restart_files = [f for f in os.listdir(nemo_rst) if - re.findall(r'.+_\d{8}_restart(_\d+)?\.nc', f) or - re.findall(r'.+_\d{8}_restart_icb(_\d+)?\.nc', f)] - nemo_restart_files.sort() - if nemo_restart_files: - latest_nemo_dump = nemo_rst + '/' + nemo_restart_files[-1] - else: - latest_nemo_dump = 'unset' - - nemo_init_dir = '.' - - # We need to ensure any lingering NEMO or iceberg retarts from - # previous runs are removed to ensure they're not accidentally - # picked up if we're starting from climatology on this occasion. - common.remove_file('restart.nc') - common.remove_file('restart_icebergs.nc') - common.remove_file('restart_icb.nc') - - if common_env['CONTINUE'] == 'false': - # This is a new run - sys.stdout.write('[INFO] New nemo run\n') - if os.path.isfile(latest_nemo_dump): - #os.path.isfile will return true for symbolic links aswell - sys.stdout.write('[INFO] Removing old NEMO restart data\n') - for file_path in glob.glob(nemo_rst+'/*restart*'): - common.remove_file(file_path) - for file_path in glob.glob(ice_rst+'/*restart*'): - common.remove_file(file_path) - for file_path in glob.glob(nemo_rst+'/*trajectory*'): - common.remove_file(file_path) - # source our history namelist file from the current directory in case - # of first cycle - history_nemo_nl = os.path.join(nemo_init_dir, nemo_envar['NEMO_NL']) - elif os.path.isfile(latest_nemo_dump): - sys.stdout.write('[INFO] Restart data available in NEMO restart ' - 'directory %s. Restarting from previous task output\n' - % nemo_rst) - sys.stdout.write('[INFO] Sourcing namelist file from the work ' - 'directory of the previous cycle\n') - # find the previous work directory if there is one - if common_env['CONTINUE_FROM_FAIL'] == 'false': - if common_env['CNWP_SUB_CYCLING'] == 'True': - prev_workdir = common.find_previous_workdir( \ - common_env['CYLC_TASK_CYCLE_POINT'], - common_env['CYLC_TASK_WORK_DIR'], - common_env['CYLC_TASK_NAME'], - common_env['CYLC_TASK_PARAM_run']) - else: - prev_workdir = common.find_previous_workdir( \ - common_env['CYLC_TASK_CYCLE_POINT'], - common_env['CYLC_TASK_WORK_DIR'], - common_env['CYLC_TASK_NAME']) - history_nemo_nl = os.path.join(prev_workdir, nemo_envar['NEMO_NL']) - else: - history_nemo_nl = nemo_envar['NEMO_NL'] - nemo_init_dir = nemo_rst - else: - sys.stderr.write('[FAIL] No restart data available in NEMO restart ' - 'directory:\n %s\n' % nemo_rst) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - - # Strings which are different pre-NEMO4.2 and at NEMO4.2 - if int(nemo_envar['NEMO_VERSION']) < 402: - gl_step_int_match = 'rn_rdt=' - iceberg_rst_part1 = '_icebergs_' - iceberg_rst_part2 = '_restart' - iceberg_link_name = 'restart_icebergs' - else: - gl_step_int_match = 'rn_dt=' - iceberg_rst_part1 = '_' - iceberg_rst_part2 = '_restart_icb' - iceberg_link_name = 'restart_icb' - - # Any variables containing things that can be globbed will start with gl_ - gl_first_step_match = 'nn_it000=' - gl_last_step_match = 'nn_itend=' - - gl_nemo_restart_date_match = 'ln_rstdate' - gl_model_basis_time = 'nn_date0=' - - # Read values from the nemo namelist file used by the previous cycle - # (if appropriate), or the configuration namelist if this is the initial - # cycle. + nemo_restart_files, latest_nemo_dump, nemo_init_dir \ + = nemo_restart_lib.compile_nemo_restart_files(nemo_rst) - # Make sure this file exists before trying to read it since restarted models - # may have had old work directories removed for numerous reasons. - if not os.path.isfile(history_nemo_nl): - sys.stderr.write('[FAIL] Cannot find namelist file %s to extract ' - 'timestep data.\n' % history_nemo_nl) - sys.exit(error.MISSING_MODEL_FILE_ERROR) + restart_nl_path = nemo_restart_lib.setup_previous_restart_nl( + common_env, nemo_rst, ice_rst, top_rst, latest_nemo_dump) - # First timestep of the previous cycle - _, first_step_val = common.exec_subproc(['grep', gl_first_step_match, - history_nemo_nl]) + history_nemo_nl = os.path.join(restart_nl_path, nemo_envar['NEMO_NL']) - nemo_first_step = int(re.findall(r'.+=(.+),', first_step_val)[0]) - - # Last timestep of the previous cycle - _, last_step_val = common.exec_subproc(['grep', gl_last_step_match, - history_nemo_nl]) - nemo_last_step = re.findall(r'.+=(.+),', last_step_val)[0] - - # The string in the nemo time step field might have any one of - # a number of variants. e.g. "set_by_rose", "set_by_system", - # "set_by_um", etc. Hence we just check for the presence of - # purely numeric characters to see if we start from zero or not. - if nemo_last_step.isdigit(): - nemo_last_step = int(nemo_last_step) - else: - nemo_last_step = 0 - - # Determine (as an integer) the number of seconds per model timestep - _, nemo_step_int_val = common.exec_subproc(['grep', gl_step_int_match, - nemo_envar['NEMO_NL']]) - nemo_step_int = int(re.findall(r'.+=(\d*)', nemo_step_int_val)[0]) - - # If the value for nemo_rst_date_value is true then the model uses - # absolute date convention, otherwise the dump times are relative to the - # start of the model run and have an integer representation - _, nemo_rst_date_value = common.exec_subproc([ \ - 'grep', gl_nemo_restart_date_match, history_nemo_nl]) - if 'true' in nemo_rst_date_value: - nemo_rst_date_bool = True - else: - nemo_rst_date_bool = False + nemo_first_step, nemo_last_step, nemo_step_int, nemo_rst_date_bool \ + = nemo_lib.read_history_nl(history_nemo_nl) # The initial date of the model run (YYYYMMDD) nemo_ndate0 = '%04d%02d%02d' % tuple(model_basis[:3]) nemo_dump_time = "00000000" - # Get the model basis time for this run (YYYYMMDD) - _, model_basis_val = common.exec_subproc( - ['grep', gl_model_basis_time, history_nemo_nl]) - nemo_model_basis = re.findall(r'.+=(.+),', model_basis_val)[0] - if os.path.isfile(latest_nemo_dump): nemo_dump_time = re.findall(r'_(\d*)_restart', latest_nemo_dump)[0] - # Verify the dump time against cycle time if appropriate, do the - # automatic fix, and check all other restart files match - if common_env['DRIVERS_VERIFY_RST'] == 'True': - nemo_dump_time = _verify_fix_rst( - nemo_dump_time, - nemo_rst, - nemo_model_basis, nemo_step_int, nemo_last_step, - common_env['CALENDAR']) - - _verify_nemo_rst(nemo_dump_time, nemo_rst, nemo_envar['NEMO_NL'], - int(nemo_envar['NEMO_NPROC']), - int(nemo_envar['NEMO_VERSION'])) + nemo_dump_time = _verify_restart( + common_env, nemo_envar, nemo_dump_time, nemo_rst) # link restart files no that the last output one becomes next input one common.remove_file('restart.nc') - - common.remove_file('restart_ice_in.nc') + common.remove_file('restart_ice.nc') + common.remove_file('restart_trc.nc') # Sort out the processor restart files - if int(nemo_envar['NEMO_NPROC']) == 1: - sys.stderr.write('[FAIL] NEMO driver does not support the running' - ' of NEMO in serial mode\n') - sys.exit(SERIAL_MODE_ERROR) - else: + _processor_restart_files_crun( + nemo_init_dir, nemo_envar['RST_LINK_DIR'], + int(nemo_envar['NEMO_NPROC']), nemo_dump_time, common_env['RUNID']) - nemo_restart_count = 0 - ice_restart_count = 0 - iceberg_restart_count = 0 - - # Nemo has multiple processors - for i_proc in range(int(nemo_envar['NEMO_NPROC'])): - tag = str(i_proc).zfill(4) - nemo_rst_source = '%s/%so_%s_restart_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time, tag) - nemo_rst_link = 'restart_%s.nc' % tag - common.remove_file(nemo_rst_link) - - if os.path.isfile(nemo_rst_source): - os.symlink(nemo_rst_source, nemo_rst_link) - nemo_restart_count += 1 - - ice_rst_source = '%s/%so_%s_restart_ice_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time, tag) - if os.path.isfile(ice_rst_source): - ice_rst_link = 'restart_ice_in_%s.nc' % tag - common.remove_file(ice_rst_link) - os.symlink(ice_rst_source, ice_rst_link) - ice_restart_count += 1 - - iceberg_rst_source = '%s/%so%s%s%s_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], iceberg_rst_part1, - nemo_dump_time, iceberg_rst_part2, tag) - if os.path.isfile(iceberg_rst_source): - iceberg_rst_link = '%s_%s.nc' % (iceberg_link_name, tag) - common.remove_file(iceberg_rst_link) - os.symlink(iceberg_rst_source, iceberg_rst_link) - iceberg_restart_count += 1 - #endfor - - if nemo_restart_count < 1: - sys.stdout.write('[INFO] No NEMO sub-PE restarts found\n') - # We found no nemo restart sub-domain files let's - # look for a global file. - nemo_rst_source = '%s/%so_%s_restart.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time) - if os.path.isfile(nemo_rst_source): - sys.stdout.write('[INFO] Using rebuilt NEMO restart '\ - 'file: %s\n' % nemo_rst_source) - nemo_rst_link = 'restart.nc' - common.remove_file(nemo_rst_link) - os.symlink(nemo_rst_source, nemo_rst_link) - - if ice_restart_count < 1: - sys.stdout.write('[INFO] No ice sub-PE restarts found\n') - # We found no ice restart sub-domain files let's - # look for a global file. - ice_rst_source = '%s/%so_%s_restart_ice.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time) - if os.path.isfile(ice_rst_source): - sys.stdout.write('[INFO] Using rebuilt ice restart '\ - 'file: %s\n' % ice_rst_source) - ice_rst_link = 'restart_ice_in.nc' - common.remove_file(ice_rst_link) - os.symlink(ice_rst_source, ice_rst_link) - - if iceberg_restart_count < 1: - sys.stdout.write('[INFO] No iceberg sub-PE restarts found\n') - # We found no iceberg restart sub-domain files let's - # look for a global file. - iceberg_rst_source = '%s/%so%s%s%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], iceberg_rst_part1, - nemo_dump_time, iceberg_rst_part2) - if os.path.isfile(iceberg_rst_source): - sys.stdout.write('[INFO] Using rebuilt iceberg restart'\ - 'file: %s\n' % iceberg_rst_source) - iceberg_rst_nc = iceberg_link_name + '.nc' - common.remove_file(iceberg_rst_nc) - os.symlink(iceberg_rst_source, iceberg_rst_link) - - #endif (nemo_envar(NEMO_NPROC) == 1) - if nemo_rst_date_bool: - #Then nemo_dump_time has the form YYYYMMDD - pass - else: - #nemo_dump_time is relative to start of model run and is an - #integer - nemo_dump_time = int(nemo_dump_time) - completed_days = nemo_dump_time * (nemo_step_int // 86400) - sys.stdout.write('[INFO] Nemo has previously completed %i days\n' % - completed_days) - ln_restart = ".true." - restart_ctl = 2 - if common_env['CONTINUE_FROM_FAIL'] == 'true': - # This is only used for coupled NWP where we don't have dates in - # NEMO restart file names - nemo_next_step = int(nemo_dump_time)+1 - else: - nemo_next_step = nemo_last_step + 1 + ln_restart, restart_ctl, nemo_next_step \ + = _crun_timesteps(nemo_rst_date_bool, nemo_dump_time, nemo_step_int, + nemo_last_step, common_env) else: # This is an NRUN - if nemo_envar['NEMO_START'] != '': - if os.path.isfile(nemo_envar['NEMO_START']): - - os.symlink(nemo_envar['NEMO_START'], 'restart.nc') - ln_restart = ".true." - - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_START']): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_START']): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # a restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('restart_%s.nc' % proc_number) - - os.symlink(fname, 'restart_%s.nc' % proc_number) - ln_restart = ".true." - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_START'][:-3]): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_START'][:-3]): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # a restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('restart_%s.nc' % proc_number) - - os.symlink(fname, 'restart_%s.nc' % proc_number) - ln_restart = ".true." - else: - sys.stderr.write('[FAIL] file %s not found\n' % - nemo_envar['NEMO_START']) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - else: - #NEMO_START is unset - sys.stdout.write('[WARN] NEMO_START not set\n' - 'NEMO will use climatology\n') - ln_restart = ".false." - - if nemo_envar['NEMO_ICEBERGS_START'] != '': - - if os.path.isfile(nemo_envar['NEMO_ICEBERGS_START']): - - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - iceberg_rst_file = iceberg_link_name + '.nc' - common.remove_file(iceberg_rst_file) - os.symlink(nemo_envar['NEMO_ICEBERGS_START'], - iceberg_rst_file) - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_ICEBERGS_START']): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_ICEBERGS_START']): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('%s_%s.nc' % - (iceberg_rst_link, proc_number)) + # Set up NEMO Nrun and ln_restart + ln_restart = _processor_restart_files_nrun(nemo_envar, common_env, + nemo_init_dir) + restart_ctl, nemo_next_step, nemo_last_step \ + = _nrun_timesteps(nemo_first_step) - os.symlink(fname, '%s_%s.nc' % - (iceberg_rst_file, proc_number)) - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_ICEBERGS_START'][:-3]): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_ICEBERGS_START'][:-3]): - proc_number = fname.split('.')[-2][-4:] + nemo_final_step = nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, run_days, + run_length, nemo_next_step, nemo_last_step) - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('%s_%s.nc' % - (iceberg_rst_link, proc_number)) - os.symlink(fname, '%s_%s.nc' % - (iceberg_rst_link, proc_number)) - else: - sys.stderr.write('[FAIL] file %s not found\n' % - nemo_envar['NEMO_ICEBERGS_START']) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - else: - #NEMO_ICEBERGS_START unset - sys.stdout.write('[WARN] NEMO_ICEBERGS_START not set or file(s)' - ' not found. Icebergs (if switched on) will start' - ' from a state of zero icebergs\n') - restart_ctl = 0 - nemo_next_step = nemo_first_step - nemo_last_step = nemo_first_step - 1 - - if common_env['CONTINUE_FROM_FAIL'] == 'true': - #Check that the length of run is correct - #(it won't be if this is the wrong restart file) - run_start_dt = datetime.datetime(run_start[0], run_start[1], - run_start[2], run_start[3]) - model_basis_dt = datetime.datetime(model_basis[0], model_basis[1], - model_basis[2], model_basis[3]) - nemo_init_step = (run_start_dt-model_basis_dt).total_seconds() \ - /nemo_step_int - tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ - + run_length[4]*60 + run_length[5] - nemo_final_step = int((tot_runlen_sec//nemo_step_int) + nemo_init_step) - # Check that nemo_next_step is the correct number of hours to - # match LAST_DUMP_HOURS variable - steps_per_hour = 3600./nemo_step_int - last_dump_hrs = int(common_env['LAST_DUMP_HOURS']) - last_dump_step = int(nemo_init_step + last_dump_hrs*steps_per_hour) - if nemo_next_step-1 != last_dump_step: - sys.stderr.write('[FAIL] Last NEMO restarts not at correct time') - sys.exit(error.RESTART_FILE_ERROR) - else: - tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ - + run_length[4]*60 + run_length[5] - nemo_final_step = (tot_runlen_sec // nemo_step_int) + nemo_last_step - - - #Make our call to update the nemo namelist. First generate the list - #of commands - if int(nemo_envar['NEMO_VERSION']) >= 400: - # from NEMO 4.0 onwards we don't have jpnij in the namelist - update_nl_cmd = '--file %s --runid %so --restart %s --restart_ctl %s' \ - ' --next_step %i --final_step %s --start_date %s --leapyear %i' \ - ' --iproc %s --jproc %s --cpl_river_count %s --verbose' % \ - (nemo_envar['NEMO_NL'], \ - common_env['RUNID'], \ - ln_restart, \ - restart_ctl, \ - nemo_next_step, \ - nemo_final_step, \ - nemo_ndate0, \ - nleapy, \ - nemo_envar['NEMO_IPROC'], \ - nemo_envar['NEMO_JPROC'], \ - common_env['CPL_RIVER_COUNT']) - else: - update_nl_cmd = '--file %s --runid %so --restart %s --restart_ctl %s' \ - ' --next_step %i --final_step %s --start_date %s' \ - ' --leapyear %i --iproc %s --jproc %s --ijproc %s' \ - ' --cpl_river_count %s --verbose' % \ - (nemo_envar['NEMO_NL'], \ - common_env['RUNID'], \ - ln_restart, \ - restart_ctl, \ - nemo_next_step, \ - nemo_final_step, \ - nemo_ndate0, \ - nleapy, \ - nemo_envar['NEMO_IPROC'], \ - nemo_envar['NEMO_JPROC'], \ - nemo_envar['NEMO_NPROC'], \ - common_env['CPL_RIVER_COUNT']) - - update_nl_cmd = './update_nemo_nl %s' % update_nl_cmd - - # REFACTOR TO USE THE SAFE EXEC SUBPROC - update_nl_rcode, _ = common.__exec_subproc_true_shell([ \ - update_nl_cmd]) - if update_nl_rcode != 0: - sys.stderr.write('[FAIL] Error updating nemo namelist\n') - sys.exit(error.SUBPROC_ERROR) - - # We just check for the presence of T or t (as in TRUE, True or true) - # in the L_OCN_PASS_TRC value. - if ('T' in nemo_envar['L_OCN_PASS_TRC']) or \ - ('t' in nemo_envar['L_OCN_PASS_TRC']): - - sys.stdout.write('[INFO] nemo_driver: Passive tracer code is ' - 'active.\n') - - controller_mode = "run_controller" - - top_controller.run_controller(common_env, - restart_ctl, - int(nemo_envar['NEMO_NPROC']), - common_env['RUNID'], - common_env['DRIVERS_VERIFY_RST'], - nemo_dump_time, - controller_mode) - else: - sys.stdout.write('[INFO] nemo_driver: ' - 'Passive tracer code not active\n.') + # Update the NEMO namelist + update_nemo_nl.update_nl(common_env, nemo_envar, ln_restart, restart_ctl, + nemo_next_step, nemo_final_step, nemo_ndate0, + nleapy) - use_si3 = 'si3' in common_env['models'] - if use_si3: - sys.stdout.write('[INFO] nemo_driver: SI3 code is active.\n') - controller_mode = "run_controller" - si3_controller.run_controller(common_env, - restart_ctl, - int(nemo_envar['NEMO_NPROC']), - common_env['RUNID'], - common_env['DRIVERS_VERIFY_RST'], - nemo_dump_time, - controller_mode) + # Deal with submodels that have unique requirements + _run_submodel_custom(common_env, nemo_envar, ln_restart, restart_ctl) return nemo_envar def _set_launcher_command(launcher, nemo_envar): ''' - Setup the launcher command for the executable + Set up the launcher command for the executable ''' if nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'] == 'unset': ss = False @@ -832,126 +311,32 @@ def _set_launcher_command(launcher, nemo_envar): nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'] return launch_cmd -def get_ocean_resol(nemo_nl_file, run_info): - ''' - Determine the ocean resolution. - This function is only used when creating the namcouple at run time. - ''' - - # See if resolution is contained within namelists (existent of - # namelist_cfg has already been checked) - ocean_nml = f90nml.read(nemo_nl_file) - - # Check the required entries exist - if 'namcfg' not in ocean_nml: - sys.stderr.write('[FAIL] namcfg not found in namelist_cfg\n') - sys.exit(error.MISSING_OCN_RESOL_NML) - - if 'jpiglo' in ocean_nml['namcfg']: - # Resolution is contained within namelists - - if 'jpiglo' not in ocean_nml['namcfg'] or \ - 'jpjglo' not in ocean_nml['namcfg'] or \ - 'cp_cfg' not in ocean_nml['namcfg'] or \ - 'jp_cfg' not in ocean_nml['namcfg']: - sys.stderr.write('[FAIL] cp_cfg, jp_cfg, jpiglo or jpjglo are ' - 'missing from namelist namcf in namelist_cfg\n') - sys.exit(error.MISSING_OCN_RESOL) - - # Check it is on orca grid - if ocean_nml['namcfg']['cp_cfg'] != 'orca': - sys.stderr.write('[FAIL] we can currently only handle the ' - 'ORCA grid\n') - sys.exit(error.NOT_ORCA_GRID) - - # Check this is a grid we recognise - if ocean_nml['namcfg']['jp_cfg'] == 25: - run_info['OCN_grid'] = 'orca025' - else: - run_info['OCN_grid'] = 'orca' + str(ocean_nml['namcfg']['jp_cfg']) - - # Store the ocean resolution - run_info['OCN_resol'] = [ocean_nml['namcfg']['jpiglo'], - ocean_nml['namcfg']['jpjglo']] - - else: - # Resolution should be contained within a domain_cfg netCDF file. - # Rather than read this file, assume resolution is declared. - if 'OCN_grid' not in run_info: - sys.stderr.write('[FAIL] it is necessary to declare the ocean ' - 'resolution by setting the OCN_RES environment ' - 'variable.') - sys.exit(error.NOT_DECLARE_OCN_RES) - else: - # Determine the ocean resolution - if run_info['OCN_grid'] not in OCEAN_RESOLS: - sys.stderr.write('[FAIL] the ocean resolution for %s is ' - 'unknown' % run_info['OCN_grid']) - sys.exit(error.UNKNOWN_OCN_RESOL) - else: - run_info['OCN_resol'] = [OCEAN_RESOLS[run_info['OCN_grid']][0], - OCEAN_RESOLS[run_info['OCN_grid']][1]] - - return run_info -def _sent_coupling_fields(nemo_envar, run_info): +def _write_output_file_to_stdout(out_file_path, error_count=0): ''' - Write the coupling fields sent from NEMO into model_snd_list. - This function is only used when creating the namcouple at run time. + Write ocean output file to standard out, and append the number of errors + to error_count (if present), or count and return number of errors in this + file if not present. The output files have unicode encoding, and + writing to standard out can't handle this, so we filter out any non ascii + characters from the line ''' - # Check that file specifying the coupling fields sent from - # NEMO is present - if not os.path.exists('OASIS_OCN_SEND'): - sys.stderr.write('[FAIL] OASIS_OCN_SEND is missing.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND) - - # Add toyatm to our list of executables - if not 'exec_list' in run_info: - run_info['exec_list'] = [] - run_info['exec_list'].append('toyoce') - - # Store ocean resolution if it is provided - if nemo_envar['OCN_RES']: - run_info['OCN_grid'] = nemo_envar['OCN_RES'] - - # Store the nemo version - if nemo_envar['NEMO_VERSION']: - run_info['NEMO_VERSION'] = nemo_envar['NEMO_VERSION'] - - # Determine the ocean resolution - run_info = get_ocean_resol(nemo_envar['NEMO_NL'], run_info) - - # If using the default coupling option, we'll need to read the - # NEMO namelist later - run_info['nemo_nl'] = nemo_envar['NEMO_NL'] - - # Read the namelist - oasis_nml = f90nml.read('OASIS_OCN_SEND') - - # Check we have the expected information - if 'oasis_ocn_send_nml' not in oasis_nml: - sys.stderr.write('[FAIL] namelist oasis_ocn_send_nml is ' - 'missing from OASIS_OCN_SEND.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND_NML) - if 'oasis_ocn_send' not in oasis_nml['oasis_ocn_send_nml']: - sys.stderr.write('[FAIL] entry oasis_ocn_send is missing ' - 'from namelist oasis_ocn_send_nml in ' - 'OASIS_OCN_SEND.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND) - - # Create a list of fields sent from OCN - import write_namcouple - model_snd_list = \ - write_namcouple.add_to_cpl_list( \ - 'OCN', False, 0, - oasis_nml['oasis_ocn_send_nml']['oasis_ocn_send']) - - return run_info, model_snd_list - -def write_ocean_out_to_stdout(): + sys.stdout.write('[INFO] Ocean output from file %s\n' % out_file_path) + with open(out_file_path, 'r', encoding='utf8') as n_out: + for line in n_out: + # remove all non ascii characters + line = ''.join(i for i in line if ord(i) < 128) + sys.stdout.write(line) + if 'E R R O R' in line: + error_count += 1 + return error_count + + +def _write_ocean_out_to_stdout(): ''' - Write the contents of ocean.output to stnadard out + Write the contents of ocean.output to standard out, and determine if + there was an error in this run ''' + error_count = 0 # append the ocean output and solver stat file to standard out. Use an # iterator to read the files, incase they are too large to fit into # memory. Try to find both the NEMO 3.6 and NEMO 4.0 solver files for @@ -966,17 +351,23 @@ def write_ocean_out_to_stdout(): # The output file from NEMO4.0 has some suspect utf8 encoding, # this try/except will handle it if os.path.isfile(nemo_output_file): - sys.stdout.write('[INFO] Ocean output from file %s\n' % - nemo_output_file) - with open(nemo_output_file, 'r', encoding='utf-8') as n_out: - for line in n_out: - try: - sys.stdout.write(line) - except UnicodeEncodeError: - pass + error_count = _write_output_file_to_stdout( + nemo_output_file, error_count) else: sys.stdout.write('[INFO] Nemo output file %s not avaliable\n' % nemo_output_file) + return error_count + +def _copy_nl_end_of_run(nemo_envar_fin, nemo_rst, top_rst): + ''' + Copy namelist files at the end of run so next cycle can find them. We assume + nemo_rst will always exist + ''' + if os.path.isdir(nemo_rst): + shutil.copy(nemo_envar_fin['NEMO_NL'], nemo_rst) + if top_rst: + if os.path.isdir(top_rst): + shutil.copy(nemo_envar_fin['TOP_NL'], top_rst) def _finalize_executable(common_env): ''' @@ -987,10 +378,8 @@ def _finalize_executable(common_env): sys.stdout.write('[INFO] finalizing NEMO\n') sys.stdout.write('[INFO] running finalize in %s\n' % os.getcwd()) - write_ocean_out_to_stdout() + error_count = _write_ocean_out_to_stdout() - _, error_count = common.__exec_subproc_true_shell([ \ - 'grep "E R R O R" ocean.output | wc -l']) if int(error_count) >= 1: sys.stderr.write('[FAIL] An error has been found with the NEMO run.' ' Please investigate the ocean.output file for more' @@ -999,31 +388,12 @@ def _finalize_executable(common_env): # move the nemo namelist to the restart directory to allow the next cycle # to pick it up - nemo_envar_fin = dr_env_lib.env_lib.LoadEnvar() - nemo_envar_fin = dr_env_lib.env_lib.load_envar_from_definition( - nemo_envar_fin, dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL) - nemo_rst = _get_nemorst(nemo_envar_fin['NEMO_NL']) - if os.path.isdir(nemo_rst) and \ - os.path.isfile(nemo_envar_fin['NEMO_NL']): - shutil.copy(nemo_envar_fin['NEMO_NL'], nemo_rst) - - # The only way to check if TOP is active is by checking the - # passive tracer env var. - - # Check whether we need to finalize the TOP controller - if ('T' in nemo_envar_fin['L_OCN_PASS_TRC']) or \ - ('t' in nemo_envar_fin['L_OCN_PASS_TRC']): - - sys.stdout.write('[INFO] nemo_driver: Finalize TOP controller.') - - controller_mode = "finalize" - top_controller.run_controller([], [], [], [], [], [], controller_mode) - - use_si3 = 'si3' in common_env['models'] - if use_si3: - sys.stdout.write('[INFO] nemo_driver: Finalise SI3 controller\n') - controller_mode = "finalize" - si3_controller.run_controller([], [], [], [], [], [], controller_mode) + nemo_envar_fin, models = nemo_lib.load_environment_variables( + 'final', common_env['models']) + common_env['models'] = models + nemo_rst, _, _, top_rst = nemo_lib.read_current_cycle_nl( + common_env, nemo_envar_fin) + _copy_nl_end_of_run(nemo_envar_fin, nemo_rst, top_rst) def run_driver(common_env, mode, run_info): ''' @@ -1032,13 +402,13 @@ def run_driver(common_env, mode, run_info): ''' if mode == 'run_driver': exe_envar = _setup_executable(common_env) - launch_cmd = _set_launcher_command(common_env['ROSE_LAUNCHER'], - exe_envar) + launch_cmd = _set_launcher_command(common_env['ROSE_LAUNCHER'], exe_envar) if run_info['l_namcouple']: model_snd_list = None else: run_info, model_snd_list = \ - _sent_coupling_fields(exe_envar, run_info) + nemo_runtime_namcouple.sent_coupling_fields( + exe_envar, run_info) elif mode == 'finalize': _finalize_executable(common_env) exe_envar = None @@ -1046,7 +416,7 @@ def run_driver(common_env, mode, run_info): model_snd_list = None elif mode == 'failure': # subset of operations of the model fails - write_ocean_out_to_stdout() + _write_ocean_out_to_stdout() exe_envar = None launch_cmd = None model_snd_list = None diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 4e4b690c..ead3c6e1 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -1,6 +1,6 @@ #!/usr/bin/env bash #*****************************COPYRIGHT****************************** -# (C) Crown copyright 2021-2025 Met Office. All rights reserved. +# (C) Crown copyright 2021 Met Office. All rights reserved. # # Use, duplication or disclosure of this code is subject to the restrictions # as set forth in the licence. If no licence has been raised with this copy @@ -37,42 +37,14 @@ export DRIVERS_FINALISED=false L_MCT_VALIDATE=${L_MCT_VALIDATE:=false} -# Copy scripts to work directory so they can be found. We use the dependency -# checker script so we only copy the files we need. -if [[ $DRIVER_EXTRACT_DIR == *"fcm_make"* ]] ; then - # fcm_make infrastructure - extract_path=extract/drivers -else - extract_path= -fi -FILES_TO_INSTALL=$($DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_dependencies.py --extract-directory $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers) -for file in $FILES_TO_INSTALL; do - # handle any packages - package_name=$(dirname "${file}") - if [[ $package_name != '.' && ! -d $package_name ]]; then - mkdir $package_name - fi - # as package_name will default to . then this will handle all cases - cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/$file $package_name -done +# Copy scripts to work directory so they can be found. +cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/*py ./ +cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/run_model ./ +cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/link_drivers ./ +cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/OASIS_fields ./ -# If the run uses LFRic, and environment variable NAMCOUPLE_STATIC is .false. -# then we need to use the NGMS_utils libraries. NAMCOUPLE_STATIC is not a -# compulsory environment variable, so we set a default for backward -# compatibility -NAMCOUPLE_STATIC=${NAMCOUPLE_STATIC:=True} -if [[ "$models" == *lfric* && "$NAMCOUPLE_STATIC" == .false. ]]; then - cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_namcouple_gen/*py ./ - cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_suite_lib/*py ./ -fi - -EXTRA_LINK_DRIVERS_INFO=$( echo ${EXTRA_LINK_DRIVERS_INFO:-} | tr '[:upper:]' '[:lower:]') -if [[ ${EXTRA_LINK_DRIVERS_INFO:-} == "true" ]]; then - stat ./ - stat ./link_drivers - lfs getstripe --mdt-index ./link_drivers - lfs getstripe ./link_drivers -fi +# Copy our packages to work directory so they can be found +cp -r $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/dr_*_lib ./ # Run our drivers, this produces two command files # driver_envar containing environment variables to be exported @@ -85,8 +57,6 @@ echo '[DRIVER_TEST_SCRIPT] Drivers successfully linked' # ensure that any dynamic namcouple setup, namelist setup, and file moving # had been completed if [ "$L_MCT_VALIDATE" = True ]; then - echo "Running MCT validate" - cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_utilities/mct_validate/mct_validate.py ./ ./mct_validate.py fi @@ -111,24 +81,21 @@ done # Export the enviromnent variables from all the drivers source driver_envar -# Run the MPI launcher command and provide timing information if required +# Run the aprun command and provide timing information if required start=$(date +%s) source driver_cmd end=$(date +%s) -time_in_launcher=$((end-start)) -echo "[DRIVER_TEST_SCRIPT] $time_in_launcher seconds are spent in the" -echo "[DRIVER_TEST_SCRIPT] MPI Launcher" +time_in_aprun=$((end-start)) +echo "[DRIVER_TEST_SCRIPT] $time_in_aprun seconds are spent in APRUN" -#export time_in_launcher variable so the cpmip controller can find it -#if required -export time_in_launcher +#export time_in_aprun variable so the cpmip controller can find it if required +export time_in_aprun # Run the drivers in finalize mode (after completion of the model). This deals # with model standard out / standard error, and also with any file details # required prior to the start of the next model cycle (if applicable). If -# the MPI launcher has failed, we run the drivers in a stripped down failure -# mode +# aprun has failed, we run the drivers in a stripped down failure mode echo '[DRIVER_TEST_SCRIPT] Finalizing drivers' ./link_drivers --finalize; export DRIVERS_FINALISED=true diff --git a/Coupled_Drivers/unittests/test.py b/Coupled_Drivers/unittests/test.py index ae4d6370..08e229b0 100755 --- a/Coupled_Drivers/unittests/test.py +++ b/Coupled_Drivers/unittests/test.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2022-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -34,18 +34,13 @@ def main(): 'all': 'test*.py', 'env_lib': 'test_env_lib.py', 'aprun_command': 'test_aprun_command_construction.py', - 'mct': 'test_mct_driver.py', - 'lfric': 'test_lfric_driver.py', - 'xios': 'test_xios_driver.py', - 'dependency_checker': 'test_driver_dependencies.py', - 'cpmip': 'test_cpmip*py', - 'nemo_driver': 'test_nemo_driver.py', - 'um_driver': 'test_um_driver.py', - 'cice_driver': 'test_cice_driver.py', - 'rivers': 'test_rivers_driver.py', - 'common': 'test_common.py', - 'namcouple': 'test_update_namcouple.py', - 'link': 'test_link_drivers.py', + 'ocn_lib': 'test_ocn_lib.py', + 'nemo_nl': 'test_update_nemo_nl.py', + 'nemo_driver': 'test_nemo_*.py', + 'nemo_lib': 'test_nemo_*lib*.py', + 'nemo_restart_lib': 'test_nemo_restart_lib*.py', + 'common': 'test_common*.py', + 'dev': 'test_ocn_lib.py', } parser = argparse.ArgumentParser( @@ -79,7 +74,7 @@ def main(): test_rtn = unittest.TextTestRunner(buffer=True).run(test_suite) rtn_code += len(test_rtn.failures) + len(test_rtn.errors) - sys.exit(rtn_code) + exit(rtn_code) if __name__ == '__main__': main() diff --git a/Coupled_Drivers/unittests/test_aprun_command_construction.py b/Coupled_Drivers/unittests/test_aprun_command_construction.py index 62402919..12b0e4a7 100644 --- a/Coupled_Drivers/unittests/test_aprun_command_construction.py +++ b/Coupled_Drivers/unittests/test_aprun_command_construction.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,12 +18,8 @@ Tests construction of aprun command. ''' import unittest -try: - import unittest.mock as mock -except ImportError: - import mock +import unittest.mock as mock import os -import copy from io import StringIO import common @@ -32,7 +28,6 @@ import nemo_driver import xios_driver import jnr_driver -import lfric_driver #pylint: disable=protected-access @@ -80,7 +75,6 @@ def test_set_aprun_options_single_proc(self): self.assertEqual(opts, expected_output) - class TestUMDriver(unittest.TestCase): ''' Tests for um_driver.py ''' @@ -202,77 +196,6 @@ def test_set_launcher_command_driver_construction_not_aprun(self): self.assertEqual(um_envar["ROSE_LAUNCHER_PREOPTS_UM"], \ expected_output_preopts) - -class TestLFRicDriver(unittest.TestCase): - '''Tests for lfric driver''' - - def setUp(self): - ''' - Setup a common lfric_envar object - ''' - self.lfric_envar = dr_env_lib.env_lib.LoadEnvar() - self.lfric_envar.add('LFRIC_LINK', 'lfric.exe') - self.lfric_envar.add('CONFIG_NL_PATH_LFRIC', 'config_nl_path') - - def test_set_launcher_command_suite_construction(self): - '''Test LFRic driver _set_launcher_command() returns a string with the - correct values if ROSE_LAUNCHER_PREOPTS_LFRIC is set as an environment - variable regardless of the launcher''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'string of preopts') - - expected_output_preopts = "'string of preopts'" - expected_cmd = 'string of preopts ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command(None, - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - - @mock.patch('lfric_driver.common.set_aprun_options') - def test_set_launcher_command_not_aprun(self, mock_set_aprun): - '''Test LFRic driver _set_launcher_command returns no preopts if - ROSE_LAUNCHER_PREOPTS_LFRIC is 'unset', and the launcher is not - aprun''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'unset') - - expected_output_preopts = "''" - expected_cmd = ' ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command('not_aprun', - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - mock_set_aprun.assert_not_called() - - @mock.patch('lfric_driver.common.set_aprun_options') - def test_set_launcher_command_aprun(self, mock_set_aprun): - '''Test LFRic driver _set_launcher_command returns custom preopts if - ROSE_LAUNCHER_PREOPTS_LFRIC is 'unset', and the launcher is - aprun''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'unset') - lfric_envar.add('LFRIC_NPROC', 'lfric_nproc') - lfric_envar.add('LFRIC_NODES', 'lfric_nodes') - lfric_envar.add('OMPTHR_LFRIC', 'lfric_threads') - lfric_envar.add('LFRICHYPERTHREADS', 'lfric_hyper') - - mock_set_aprun.return_value = 'custom_preopts' - - expected_output_preopts = "'custom_preopts'" - expected_cmd = 'custom_preopts ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command('aprun', - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - mock_set_aprun.assert_called_once_with( - 'lfric_nproc', 'lfric_nodes', 'lfric_threads', 'lfric_hyper', False) - - - - class TestNEMODriver(unittest.TestCase): ''' Tests for nemo_driver.py ''' @@ -563,8 +486,8 @@ def test_setup_executable_none(self, _): self._set_xios_nproc() with mock.patch('sys.stdout', new=StringIO()) as mock_out, \ - mock.patch('sys.stderr', new=StringIO()) as _, \ - mock.patch.dict(os.environ, self.xios_envar_dict): + mock.patch('sys.stderr', new=StringIO()) as mock_err, \ + mock.patch.dict(os.environ, self.xios_envar_dict): xios_driver._setup_executable(self.common_env_dict) self.assertRegex(mock_out.getvalue(), \ diff --git a/Coupled_Drivers/unittests/test_envar_lib.py b/Coupled_Drivers/unittests/test_envar_lib.py index 2a23cfa6..91e3702e 100644 --- a/Coupled_Drivers/unittests/test_envar_lib.py +++ b/Coupled_Drivers/unittests/test_envar_lib.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -14,12 +14,7 @@ ''' import unittest -try: - # mock is integrated into unittest as of Python 3.3 - import unittest.mock as mock -except ImportError: - # mock is a standalone package (back-ported) - import mock +import unittest.mock as mock import io import os From f2bdff5ea8d2018c7a13e31f41b99c38973f9d86 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 12 May 2026 09:49:09 +0100 Subject: [PATCH 02/47] Typo correction and unused function removal --- Coupled_Drivers/common.py | 28 +++------------------------- Coupled_Drivers/run_model | 4 ++-- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index 57ec1578..bad96f4d 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -80,7 +80,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): ''' Find the work directory for the previous cycle. Takes as argument the current cyclepoint, the path to the current work directory, and - the current taskname, a value specifying multiple tasks within + the current taskname, a value specifying multiple tasks within same cycle (e.g. coupled_run1, coupled_run2) as used in coupled NWP and returns an absolute path. ''' @@ -97,7 +97,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): return previous_workdir - else: + else: cyclesdir = os.sep.join(workdir.split(os.sep)[:-2]) #find the work directory for the previous cycle work_cycles = os.listdir(cyclesdir) @@ -118,7 +118,7 @@ def find_previous_workdir(cyclepoint, workdir, taskname, task_param_run=None): sys.stderr.write('[FAIL] Can not find previous work directory for' ' task %s\n' % taskname) sys.exit(error.MISSING_DRIVER_FILE_ERROR) - + return os.path.join(cyclesdir, previous_task_cycle, taskname) @@ -186,28 +186,6 @@ def remove_file(filename): else: return False -def remove_trailing_slash(path_str): - ''' - Takes a string of a path and removes any trailing slashes and spaces if - there are any present - ''' - trailing_slash_regex = r'[\w/]*\w+([\s/]+)$' - match = re.match(trailing_slash_regex, path_str) - if match: - num_trailing_slashes = len(match.group(1)) - return path_str[:-num_trailing_slashes] - return path_str - - -def remove_trailing_slash_orig(path_str): - ''' - Takes a string of a path and removes the trailing slash if there is one - present - ''' - if path_str[-1] == '/': - return path_str[:-1] - return path_str - def setup_runtime(common_env): ''' Set up model run length in seconds based on the model suite diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index ead3c6e1..20037499 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -27,7 +27,7 @@ echo $CYLC_SUITE_RUN_DIR # compatibility, and future flexibility DRIVER_EXTRACT_DIR=${DRIVER_EXTRACT_DIR:=$CYLC_SUITE_SHARE_DIR/fcm_make_drivers} -echo "[DRIVER_TEST_SCRIPT] DRIVER_EXTRACT_DIR is $DRIVER_EXTRACT_DIR" +echo "[DRIVER_TEST_SCRIPT] DRIVER_EXTRACT_DIR is $DRIVER_EXTRACT_DIR" # This lets us set up a failure mode export DRIVERS_FINALISED=false @@ -79,7 +79,7 @@ for S in $SIGNALS; do trap 'FINALLY' $S done -# Export the enviromnent variables from all the drivers +# Export the environment variables from all the drivers source driver_envar # Run the aprun command and provide timing information if required start=$(date +%s) From 5e5f650315c1bd6eabf2d8124ca7a68db1eb193f Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 12 May 2026 10:26:48 +0100 Subject: [PATCH 03/47] Replace aprun for MPI launcher --- Coupled_Drivers/run_model | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 20037499..50802a78 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -81,21 +81,21 @@ done # Export the environment variables from all the drivers source driver_envar -# Run the aprun command and provide timing information if required +# Run the MPI launcher command and provide timing information if required start=$(date +%s) source driver_cmd end=$(date +%s) -time_in_aprun=$((end-start)) -echo "[DRIVER_TEST_SCRIPT] $time_in_aprun seconds are spent in APRUN" +time_in_launcher=$((end-start)) +echo "[DRIVER_TEST_SCRIPT] $time_in_launcher seconds are spent in MPI Launcher" #export time_in_aprun variable so the cpmip controller can find it if required -export time_in_aprun +export time_in_launcher # Run the drivers in finalize mode (after completion of the model). This deals # with model standard out / standard error, and also with any file details # required prior to the start of the next model cycle (if applicable). If -# aprun has failed, we run the drivers in a stripped down failure mode +# MPI launcher has failed, we run the drivers in a stripped down failure mode echo '[DRIVER_TEST_SCRIPT] Finalizing drivers' ./link_drivers --finalize; export DRIVERS_FINALISED=true From 42113250e434c180ac85c7bd637f648d89f88e49 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 12 May 2026 11:26:55 +0100 Subject: [PATCH 04/47] copy in relevant files using dependency checker --- Coupled_Drivers/run_model | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 50802a78..1e6f7e07 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -37,14 +37,42 @@ export DRIVERS_FINALISED=false L_MCT_VALIDATE=${L_MCT_VALIDATE:=false} -# Copy scripts to work directory so they can be found. -cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/*py ./ -cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/run_model ./ -cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/link_drivers ./ -cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/OASIS_fields ./ - -# Copy our packages to work directory so they can be found -cp -r $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/dr_*_lib ./ +# Copy scripts to work directory so they can be found. We use the dependency +# checker script so we only copy the files we need. +if [[ $DRIVER_EXTRACT_DIR == *"fcm_make"* ]] ; then + # fcm_make infrastructure + extract_path=extract/drivers +else + extract_path= +fi +FILES_TO_INSTALL=$($DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_dependencies.py --extract-directory $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers) +for file in $FILES_TO_INSTALL; do + # handle any packages + package_name=$(dirname "${file}") + if [[ $package_name != '.' && ! -d $package_name ]]; then + mkdir $package_name + fi + # as package_name will default to . then this will handle all cases + cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/$file $package_name +done + +# If the run uses LFRic, and environment variable NAMCOUPLE_STATIC is .false. +# then we need to use the NGMS_utils libraries. NAMCOUPLE_STATIC is not a +# compulsory environment variable, so we set a default for backward +# compatibility +NAMCOUPLE_STATIC=${NAMCOUPLE_STATIC:=True} +if [[ "$models" == *lfric* && "$NAMCOUPLE_STATIC" == .false. ]]; then + cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_namcouple_gen/*py ./ + cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_suite_lib/*py ./ +fi + +EXTRA_LINK_DRIVERS_INFO=$( echo ${EXTRA_LINK_DRIVERS_INFO:-} | tr '[:upper:]' '[:lower:]') +if [[ ${EXTRA_LINK_DRIVERS_INFO:-} == "true" ]]; then + stat ./ + stat ./link_drivers + lfs getstripe --mdt-index ./link_drivers + lfs getstripe ./link_drivers +fi # Run our drivers, this produces two command files # driver_envar containing environment variables to be exported From 2e65232324a160c654fc71a19e9a00f1fc9d0bc7 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 12 May 2026 11:50:14 +0100 Subject: [PATCH 05/47] Add mocilib to source extraction --- Coupled_Drivers/run_model | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 1e6f7e07..8b69f3a0 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -74,6 +74,9 @@ if [[ ${EXTRA_LINK_DRIVERS_INFO:-} == "true" ]]; then lfs getstripe ./link_drivers fi +# Add MOCIlib +cp $DRIVER_EXTRACT_DIR/$extract_path/mocilib/*py ./ + # Run our drivers, this produces two command files # driver_envar containing environment variables to be exported # driver_cmd containing a command for the system launcher From d1ca74721bf830bdea03d9c06b72c7414dfc76c6 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 10:25:43 +0100 Subject: [PATCH 06/47] Update copyright in run model --- Coupled_Drivers/run_model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 8b69f3a0..0c8cc6b1 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -1,6 +1,6 @@ #!/usr/bin/env bash #*****************************COPYRIGHT****************************** -# (C) Crown copyright 2021 Met Office. All rights reserved. +# (C) Crown copyright 2021-2026 Met Office. All rights reserved. # # Use, duplication or disclosure of this code is subject to the restrictions # as set forth in the licence. If no licence has been raised with this copy From 64107827fc9018c07293b1160cbd8bbd1be2c8d3 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 10:29:00 +0100 Subject: [PATCH 07/47] add new mct setup --- Coupled_Drivers/run_model | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 0c8cc6b1..4ee47e73 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -88,6 +88,8 @@ echo '[DRIVER_TEST_SCRIPT] Drivers successfully linked' # ensure that any dynamic namcouple setup, namelist setup, and file moving # had been completed if [ "$L_MCT_VALIDATE" = True ]; then + echo "Running MCT validate" + cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_utilities/mct_validate/mct_validate.py ./ ./mct_validate.py fi From 627f7c62b10760391265b9fbcef0800234f8accd Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 11:08:34 +0100 Subject: [PATCH 08/47] Add um_tools to ex1a site --- rose-stem/site/meto_ex1a.cylc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rose-stem/site/meto_ex1a.cylc b/rose-stem/site/meto_ex1a.cylc index f79a8e60..ee9f386d 100644 --- a/rose-stem/site/meto_ex1a.cylc +++ b/rose-stem/site/meto_ex1a.cylc @@ -1,4 +1,4 @@ -{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools;' %} +{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools; module load um_tools' %} [[HPC_RESOURCE]] env-script = "eval $(rose task-env)" @@ -94,8 +94,8 @@ DATA_INTENSITY = true COMPLEXITY = false TOTAL_POWER_CONSUMPTION_NEEDED = 4.8 - NODES_IN_HPC_NEEDED = 12932 - [[[directives]]] + NODES_IN_HPC_NEEDED = 12932 + [[[directives]]] -l select = '{{ATMOS_NODES}}:ncpus=256:mpiprocs={{ATMOS_MPIPROC_PN}}+{{OCEAN_NODES}}:ncpus=256:mpiprocs={{OCEAN_PPN}}+{{XIOS_NODES}}:ncpus=256:mpiprocs={{XIOS_NPROC}}' [[RECON_RESOURCE]] From 710ff5c5ba1489c9c50ebd69c918bc0b9ed19ec4 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 11:50:05 +0100 Subject: [PATCH 09/47] Add terminator to module command --- rose-stem/site/meto_ex1a.cylc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/site/meto_ex1a.cylc b/rose-stem/site/meto_ex1a.cylc index ee9f386d..567b5631 100644 --- a/rose-stem/site/meto_ex1a.cylc +++ b/rose-stem/site/meto_ex1a.cylc @@ -1,4 +1,4 @@ -{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools; module load um_tools' %} +{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools; module load um_tools;' %} [[HPC_RESOURCE]] env-script = "eval $(rose task-env)" From 688a6668d6dc4ace0b4e4ad560b52e93d9de740b Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 13:21:06 +0100 Subject: [PATCH 10/47] Remove redundant methods --- Coupled_Drivers/unittests/test_link_drivers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Coupled_Drivers/unittests/test_link_drivers.py b/Coupled_Drivers/unittests/test_link_drivers.py index a51c9183..88bc3d73 100644 --- a/Coupled_Drivers/unittests/test_link_drivers.py +++ b/Coupled_Drivers/unittests/test_link_drivers.py @@ -62,12 +62,6 @@ class TestChecks(unittest.TestCase): Test the private check methods. ''' - def setUp(self): - pass - - def tearDown(self): - pass - def test_check_drivers_single(self): ''' Assert single model driver is present From ea9ff201269a622f20c979399330a0776f9e1bdb Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 13:43:16 +0100 Subject: [PATCH 11/47] Update check drivers function --- Coupled_Drivers/link_drivers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Coupled_Drivers/link_drivers b/Coupled_Drivers/link_drivers index d7dd6be2..f88d265c 100755 --- a/Coupled_Drivers/link_drivers +++ b/Coupled_Drivers/link_drivers @@ -44,8 +44,8 @@ def _check_drivers(models): ''' sys.stdout.write('MODELS %s\n' % models) - models = [model for model in models.split(' ') if model not in SUBMODELS] - for model in models: + models = models.split('') + for model in models[:]: drivername = '%s_driver.py' % model if not os.path.isfile(drivername): # check to see if this is a controller rather than a driver From b3303d51a3c1567c8c9212edbef281ecbf5303cc Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 13:46:59 +0100 Subject: [PATCH 12/47] Add MPI usage to link drivers --- Coupled_Drivers/link_drivers | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Coupled_Drivers/link_drivers b/Coupled_Drivers/link_drivers index f88d265c..f57e5947 100755 --- a/Coupled_Drivers/link_drivers +++ b/Coupled_Drivers/link_drivers @@ -121,28 +121,32 @@ def _setup_common(): return common_env -def _export_cmd_str_XC40(launchcmds, common_env): +def _export_cmd_str(launchcmds, common_env): ''' - Create an aprun launch command for the coupled configuration on the - XC40 + Create an MPI launch command for the model configuration ''' + message_str = '[LINK_DRIVERS] attemting to run with command: %s\n' + # If a custom command is used we write that straight into cmd variable + # rather than constructing the command string + if common_env['CUSTOM_LAUNCH_COMMAND']: + cmd = common_env['CUSTOM_LAUNCH_COMMAND'] + sys.stdout.write(message_str % cmd) + return cmd + cmd = '%s ' % common_env['ROSE_LAUNCHER'] for launchcmd in launchcmds: if launchcmd != '': cmd += launchcmd+' : ' - sys.stdout.write('[LINK_DRIVERS] attemting to run with command: %s\n' - % cmd[:-3]) + sys.stdout.write(message_str % cmd[:-3]) return cmd[:-3] - - if __name__ == '__main__': - # Owing to the limitations of aprun and passing information between the - # shell and the python drivers, prepare two command files, one containing - # an aprun command to execute, the other containing a command allowing + # Owing to the limitations of the MPI launchers and passing information + # between the shell and the python drivers, prepare two command files, one + # containing a command to execute, the other containing a command allowing # environment variables to be exported try: @@ -202,6 +206,7 @@ if __name__ == '__main__': # Create the namcouple file if not run_info['l_namcouple']: + import write_namcouple write_namcouple.write_namcouple( common_env, run_info, coupling_list) @@ -209,8 +214,8 @@ if __name__ == '__main__': # environment variable definitions that can be run by the shell envar_str = dr_env_lib.env_lib.string_for_export(envinsts) - # Create the aprun command - cmd_str = _export_cmd_str_XC40(launchcmds, common_env) + # Create the MPI launcher command + cmd_str = _export_cmd_str(launchcmds, common_env) # Write the export commands for common and model specific environment # variables (envar_str) into the file driver_envar From c8bb23695ec729b23bc931765c6a2bf7cfd9d3b3 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 13:54:30 +0100 Subject: [PATCH 13/47] Correct empty separator --- Coupled_Drivers/link_drivers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Coupled_Drivers/link_drivers b/Coupled_Drivers/link_drivers index f57e5947..0046f219 100755 --- a/Coupled_Drivers/link_drivers +++ b/Coupled_Drivers/link_drivers @@ -44,7 +44,7 @@ def _check_drivers(models): ''' sys.stdout.write('MODELS %s\n' % models) - models = models.split('') + models = models.split(' ') for model in models[:]: drivername = '%s_driver.py' % model if not os.path.isfile(drivername): From 3ed3bd5e5336c68721398a321d217c386c70a763 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 14:39:18 +0100 Subject: [PATCH 14/47] Update error codes --- Coupled_Drivers/error.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/Coupled_Drivers/error.py b/Coupled_Drivers/error.py index 3b23466d..704c78c1 100644 --- a/Coupled_Drivers/error.py +++ b/Coupled_Drivers/error.py @@ -184,11 +184,17 @@ # Not on ORCA grid NOT_ORCA_GRID = 833 +# Not declared ocean resolution when creating namcouple on fly after NEMO3.6 +NOT_DECLARE_OCN_RES = 834 + +# Unknown ocean resolution +UNKNOWN_OCN_RESOL = 835 + # Missing namelist oasis_ocn_send_nml from OASIS_OCN_SEND -MISSING_OASIS_OCN_SEND_NML = 834 +MISSING_OASIS_OCN_SEND_NML = 836 # Missing entry oasis_ocn_send from namelist oasis_ocn_send_nml -MISSING_OASIS_OCN_SEND = 835 +MISSING_OASIS_OCN_SEND = 837 # Missing OASIS send file for UM MISSING_OASIS_ATM_SEND = 840 @@ -254,3 +260,15 @@ # Not found the coupling namelist in namelist_cfg MISSING_NAMSBC_CPL = 871 + +# Missing the JULES river resolution namelist +MISSING_RIVER_RESOL_NML = 872 + +# Missing the JULES river resolution parameters +MISSING_RIVER_RESOL = 873 + +# Missing OASIS send file for JULES river +MISSING_OASIS_RIV_SEND = 874 + +# Missing namelist oasis_riv_send_nml from OASIS_RIV_SEND +MISSING_OASIS_RIV_SEND_NML = 875 From f66144688b5f511082e70d5a2b7f99aca8fd42e6 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 14:41:43 +0100 Subject: [PATCH 15/47] Add import error --- Coupled_Drivers/error.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Coupled_Drivers/error.py b/Coupled_Drivers/error.py index 704c78c1..a9d026c3 100644 --- a/Coupled_Drivers/error.py +++ b/Coupled_Drivers/error.py @@ -23,6 +23,9 @@ # Python errors VERSION_ERROR = 1 +# Import Error +IMPORT_ERROR = 2 + # File I/O errors IOERROR = 100 From 0a4d4897651f601675a4aeb522499f54ff7a4e3d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 14:50:07 +0100 Subject: [PATCH 16/47] Add hist dirs to common --- Coupled_Drivers/common.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index bad96f4d..bd2415f8 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -334,3 +334,53 @@ def set_aprun_options(nproc, nodes, ompthr, hyperthreads, ss): rose_launcher_preopts = "-ss " + rose_launcher_preopts return rose_launcher_preopts + + +def _sort_hist_dirs_by_date(dir_list): + ''' + Sort a list of history directories by date + ''' + # Pattern that defines the name of the history directories, + # which contain a date of the form YYYYmmddHHMM. + pattern = r'\.(\d{12})' + + try: + dir_list.sort(key=lambda dname: datetime.datetime.strptime( + re.search(pattern, dname).group(1), '%Y%m%d%H%M')) + except AttributeError: + msg = '[FAIL] Cannot order directories: %s' % " ".join(dir_list) + sys.stderr.write(msg) + sys.exit(error.IOERROR) + + return dir_list + + +def remove_latest_hist_dir(old_hist_dir): + ''' + If a model task has failed, then removed the last created history + directory, before a new one is created, associated with the + re-attempt. + ''' + # Replace the regex pattern that defines the history directory + # name (that contains a date of the format YYYYmmddHHMM) with a + # generic pattern so that we can perform the directory glob. + history_pattern = re.sub( + r'\.\d{12}', '.????????????', old_hist_dir) + + # Find and sort the history directories, and delete + # the latest one, corresponding to the last entry in + # the list. + history_dirs = glob.glob(history_pattern) + history_dirs = _sort_hist_dirs_by_date(history_dirs) + + msg = '[INFO] Found history directories: %s \n' % ' '.join( + history_dirs) + sys.stdout.write(msg) + + latest_hist_dir = history_dirs[-1] + msg = ("[WARN] Re-attempting failed model step. \n" + "[WARN] Clearing out latest history \n" + "[WARN] directory %s. \n" % latest_hist_dir) + sys.stdout.write(msg) + + shutil.rmtree(latest_hist_dir) From e136e3dee674196f73cf6ebea7067a482db24d74 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 15:02:25 +0100 Subject: [PATCH 17/47] Rejig imports --- Coupled_Drivers/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index bd2415f8..cb553b2e 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** (C) Crown copyright 2021 Met Office. All rights reserved. @@ -18,9 +18,9 @@ Common functions and classes required by multiple model drivers ''' -#The from __future__ imports ensure compatibility between python2.7 and 3.x -from __future__ import absolute_import -import copy +import datetime +import glob +import shutil import re import os import sys From 9040656a64a291d6684a408629756978de4a1a0d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 15:09:38 +0100 Subject: [PATCH 18/47] Update driver configs --- Coupled_Drivers/fcm_make/driver.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Coupled_Drivers/fcm_make/driver.cfg b/Coupled_Drivers/fcm_make/driver.cfg index 05f3d7a4..d7f50d40 100644 --- a/Coupled_Drivers/fcm_make/driver.cfg +++ b/Coupled_Drivers/fcm_make/driver.cfg @@ -14,8 +14,8 @@ extract.ns = drivers extract.location{primary}[drivers] = fcm:moci.xm extract.location[drivers] = $driver_base${driver_rev} extract.path-excl[drivers] = / -extract.path-incl[drivers] = Coupled_Drivers +extract.path-incl[drivers] = Coupled_Drivers Utilities/NGMS_utils/ngms_namcouple_gen Utilities/NGMS_utils/ngms_suite_lib -$install_common = cice_driver.py common.py write_namcouple.py default_couplings.py error.py inc_days.py jnr_driver.py link_drivers mct_driver.py nemo_driver.py OASIS_fields run_model save_um_state.py time2days.py um_driver.py update_namcouple.py update_nemo_nl.py xios_driver.py plot_cpmip_summary.py cpmip_controller.py mct_validate.py ocn_lib.py nemo_runtime_namcouple.py nemo_driver.py +$install_common = build.target = $install_common From 6ffcb430f7f8acd99401bf18a85eb7f838d5f604 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 15:13:35 +0100 Subject: [PATCH 19/47] Update ocean environment variables --- Coupled_Drivers/dr_env_lib/ocn_cont_def.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py index e087f0b0..bd66e8ff 100644 --- a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py +++ b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py @@ -18,18 +18,17 @@ Definition of the environment variables required for the ocean controllers ''' -SI3_ENVIRONMENT_VARS_COMMON = {'SI3_NL': {'default_val': 'namelist_ice_cfg'}} -SI3_ENVIRONMENT_VARS_INITIAL = {'SI3_START': {'default_val': ''}} -SI3_ENVIRONMENT_VARS_INITIAL = {**SI3_ENVIRONMENT_VARS_COMMON, - **SI3_ENVIRONMENT_VARS_INITIAL} -SI3_ENVIRONMENT_VARS_FINAL = SI3_ENVIRONMENT_VARS_COMMON +SI3_ENVIRONMENT_VARS_INITIAL = { + 'SI3_START': {'default_val': ''}, + 'SI3_NL': {'default_val': 'namelist_ice_cfg'}} TOP_ENVIRONMENT_VARS_COMMON = { 'TOP_NL': {'default_val': 'namelist_top_cfg'}} TOP_ENVIRONMENT_VARS_INITIAL = { - 'TOP_START': {'default_val': ''}} + 'TOP_START': {'default_val': ''}, + 'CONTINUE': {'default_val': 'false'}, + 'CONTINUE_FROM_FAIL': {'default_val': 'false'}} TOP_ENVIRONMENT_VARS_INITIAL = {**TOP_ENVIRONMENT_VARS_COMMON, **TOP_ENVIRONMENT_VARS_INITIAL} -TOP_ENVIRONMENT_VARS_FINAL = TOP_ENVIRONMENT_VARS_COMMON From 7756b31d2568de4021b501535aeeec0a89dbdd6d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 15:32:28 +0100 Subject: [PATCH 20/47] Add unit test groups --- Coupled_Drivers/unittests/test.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Coupled_Drivers/unittests/test.py b/Coupled_Drivers/unittests/test.py index 08e229b0..813c8928 100755 --- a/Coupled_Drivers/unittests/test.py +++ b/Coupled_Drivers/unittests/test.py @@ -33,7 +33,6 @@ def main(): groups = { 'all': 'test*.py', 'env_lib': 'test_env_lib.py', - 'aprun_command': 'test_aprun_command_construction.py', 'ocn_lib': 'test_ocn_lib.py', 'nemo_nl': 'test_update_nemo_nl.py', 'nemo_driver': 'test_nemo_*.py', @@ -41,6 +40,18 @@ def main(): 'nemo_restart_lib': 'test_nemo_restart_lib*.py', 'common': 'test_common*.py', 'dev': 'test_ocn_lib.py', + 'mct': 'test_mct_driver.py', + 'lfric': 'test_lfric_driver.py', + 'xios': 'test_xios_driver.py', + 'dependency_checker': 'test_driver_dependencies.py', + 'cpmip': 'test_cpmip*py', + 'nemo_driver': 'test_nemo_driver.py', + 'um_driver': 'test_um_driver.py', + 'cice_driver': 'test_cice_driver.py', + 'rivers': 'test_rivers_driver.py', + 'common': 'test_common.py', + 'namcouple': 'test_update_namcouple.py', + 'link': 'test_link_drivers.py', } parser = argparse.ArgumentParser( From 41a100f5247543bbf2cf1d0a0c4c8f3e77f2fa15 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 13 May 2026 15:33:27 +0100 Subject: [PATCH 21/47] Update driver test shebang --- Coupled_Drivers/unittests/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Coupled_Drivers/unittests/test.py b/Coupled_Drivers/unittests/test.py index 813c8928..da44b34a 100755 --- a/Coupled_Drivers/unittests/test.py +++ b/Coupled_Drivers/unittests/test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** (C) Crown copyright 2021 Met Office. All rights reserved. From 9c9dd78cba3b5fbab8796cd5344da02093d0cb5b Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 09:43:40 +0100 Subject: [PATCH 22/47] Add nemo lib Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/nemo_lib.py | 259 ++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 Coupled_Drivers/nemo_lib.py diff --git a/Coupled_Drivers/nemo_lib.py b/Coupled_Drivers/nemo_lib.py new file mode 100644 index 00000000..38c0d9bf --- /dev/null +++ b/Coupled_Drivers/nemo_lib.py @@ -0,0 +1,259 @@ +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_lib.py + +DESCRIPTION + Library functions required for the NEMO driver +''' +import datetime +import os +import sys + +import common +import dr_env_lib.env_lib +import dr_env_lib.nemo_def +import dr_env_lib.ocn_cont_def +import error +import inc_days +import ocn_lib + +# +# SECTION ONE: +# +# Section one contains functions that can be called directly from other +# modules +# + +def read_current_cycle_nl(common_env, nemo_envar): + ''' + Read the current cycle namelist for each item in turn, NEMO, SI3 and TOP + ''' + nemo_rst, ln_icebergs, ice_rst, top_rst = None, None, None, None + if 'nemo' in common_env['models']: + nemo_rst, ln_icebergs = _read_current_cycle_nl_nemo( + nemo_envar['NEMO_NL']) + if 'si3' in common_env['models']: + ice_rst = _read_current_cycle_nl_si3(nemo_envar['SI3_NL']) + if 'top' in common_env['models']: + top_rst = _read_current_cycle_nl_top(nemo_envar['TOP_NL']) + return nemo_rst, ln_icebergs, ice_rst, top_rst + + + +def setup_dates(common_env): + ''' + Setup the dates for the NEMO model run + ''' + calendar = common_env['CALENDAR'] + if calendar == '360day': + calendar = '360' + nleapy = 30 + elif calendar == '365day': + calendar = '365' + nleapy = 0 + elif calendar == 'gregorian': + nleapy = 1 + else: + sys.stderr.write('[FAIL] Calendar type %s not recognised\n' % + calendar) + sys.exit(error.INVALID_EVAR_ERROR) + + #turn our times into lists of integers + model_basis = [int(i) for i in common_env['MODELBASIS'].split(',')] + run_start = [int(i) for i in common_env['TASKSTART'].split(',')] + run_length = [int(i) for i in common_env['TASKLENGTH'].split(',')] + + run_days = inc_days.inc_days(run_start[0], run_start[1], run_start[2], + run_length[0], run_length[1], run_length[2], + calendar) + return nleapy, model_basis, run_start, run_length, run_days + +def read_history_nl(history_nemo_nl): + ''' + Read and process the desired variables from the history nemo namelist + ''' + namelist = ocn_lib.ReadOcnNamelist(history_nemo_nl) + nemo_first_step, nemo_last_step, nemo_step_int, nemo_rst_date \ + = namelist.read_variables( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date') + return (int(nemo_first_step), _check_nemo_last_step(nemo_last_step), + int(nemo_step_int), string_to_boolean(nemo_rst_date)) + +def load_environment_variables(mode, models): + ''' + Load the environment variables for NEMO and SI3 into a single + container + ''' + nemo_envar = dr_env_lib.env_lib.LoadEnvar() + # The functions to check the namelists for each model + checks = {'nemo': _check_nemonl, + 'si3': _check_si3nl, + 'top': _check_topnl} + if mode == 'setup': + defs = { + 'nemo': dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL, + 'si3': dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_INITIAL, + 'top': dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_INITIAL} + elif mode == 'final': + defs = { + 'nemo': dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL, + 'si3': dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_FINAL, + 'top': dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_FINAL} + for imodel in models.split(' '): + if imodel in defs.keys(): + nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( + nemo_envar, defs[imodel]) + checks[imodel](nemo_envar) + # TOP is controlled by the environment variable 'L_OCN_PASS_TRC' rather + # than in the models list, we must add the environment variables + if 't' in nemo_envar['L_OCN_PASS_TRC'].lower(): + models = models + ' top' + nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( + nemo_envar, defs['top']) + checks['top'](nemo_envar) + return nemo_envar, models + +def string_to_boolean(nemo_rst_date): + ''' + If a string contains a string containg upper or lower case true + (or some permeatation thereof), return python boolean True. Otherwise + return False + ''' + if 'true' in nemo_rst_date.lower(): + return True + return False + + +def setup_nemo_runlen(common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, nemo_last_step): + ''' + Setup the nemo runlength for coupled NWP models with subcycle restarts + ''' + if common_env['CONTINUE_FROM_FAIL'] == 'true': + # Check the length of the run is correct + # (it won't be if this is the wrong restart file) + run_start_dt = datetime.datetime(run_start[0], run_start[1], + run_start[2], run_start[3]) + model_basis_dt = datetime.datetime(model_basis[0], model_basis[1], + model_basis[2], model_basis[3]) + nemo_init_step = (run_start_dt-model_basis_dt).total_seconds() \ + /nemo_step_int + tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ + + run_length[4]*60 + run_length[5] + nemo_final_step = int((tot_runlen_sec//nemo_step_int) + nemo_init_step) + # Check that nemo_next_step is the correct number of hours to + # match LAST_DUMP_HOURS variable + steps_per_hour = 3600./nemo_step_int + last_dump_hrs = int(common_env['LAST_DUMP_HOURS']) + last_dump_step = int(nemo_init_step + last_dump_hrs*steps_per_hour) + if nemo_next_step-1 != last_dump_step: + sys.stderr.write('[FAIL] Last NEMO restarts not at correct time\n' + ' Last completed timestep %d\n' + ' Expected next step %d\n' + ' Actual next step %d\n' + % (last_dump_step, last_dump_step+1, + nemo_next_step)) + sys.exit(error.RESTART_FILE_ERROR) + else: + tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ + + run_length[4]*60 + run_length[5] + nemo_final_step = (tot_runlen_sec // nemo_step_int) + nemo_last_step + return nemo_final_step + + +# +# SECTION TWO: +# +# Section two contains functions that are private to this module +# + +def _check_nemo_last_step(nemo_last_step): + ''' + The string in the nemo time step field might have any one of + a number of variants. e.g. "set_by_rose", "set_by_system", + "set_by_um", etc. Hence we just check for the presence of + purely numeric characters to see if we start from zero or not. + ''' + if nemo_last_step.isdigit(): + return int(nemo_last_step) + return 0 + +def _check_nemonl(envar_container): + ''' + As the environment variable NEMO_NL is required by both the setup + and finalise functions, this will be encapsulated here + ''' + # Information will be retrieved from this file during the running of the + # driver, so check it exists + if not os.path.isfile(envar_container['NEMO_NL']): + sys.stderr.write('[FAIL] Can not find the nemo namelist file %s\n' % + envar_container['NEMO_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + +def _check_si3nl(envar_container): + ''' + Check the si3 namelist file exists as information will be retrieved from + this file during the running of the driver + ''' + if not os.path.isfile(envar_container['SI3_NL']): + sys.stderr.write('[FAIL] Can not find the SI3 namelist ' + 'file %s\n' % envar_container['SI3_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + +def _check_topnl(envar_container): + ''' + Check the TOP namelist file exists as information will be retrieved from + this file during the running of the driver''' + if not os.path.isfile(envar_container['TOP_NL']): + sys.stderr.write('[FAIL] Can not find the TOP namelist ' + 'file %s\n' % envar_container['TOP_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + + +def _read_current_cycle_nl_nemo(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for NEMO + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + nemo_rst, ln_icebergs = current_nl.read_variables( + 'nemo_rst', 'ln_icebergs') + if nemo_rst: + nemo_rst = common.remove_trailing_slash(nemo_rst) + return nemo_rst, string_to_boolean(ln_icebergs) + +def _read_current_cycle_nl_si3(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for SI3 + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + ice_rst = current_nl.read_variables('ice_rst') + if ice_rst: + ice_rst = common.remove_trailing_slash(ice_rst) + sys.stdout.write('[INFO] NEMO running with SI3 submodel\n') + return ice_rst + +def _read_current_cycle_nl_top(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for TOP + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + top_rst = current_nl.read_variables('top_rst') + if top_rst: + top_rst = common.remove_trailing_slash(top_rst) + sys.stdout.write('[INFO] NEMO running with TOP submodel\n') + return top_rst From 11173b73dfe31b97be355c7d561041c59df85a99 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 09:46:50 +0100 Subject: [PATCH 23/47] Add nemo restart lib Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/nemo_restart_lib.py | 181 ++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 Coupled_Drivers/nemo_restart_lib.py diff --git a/Coupled_Drivers/nemo_restart_lib.py b/Coupled_Drivers/nemo_restart_lib.py new file mode 100644 index 00000000..84969643 --- /dev/null +++ b/Coupled_Drivers/nemo_restart_lib.py @@ -0,0 +1,181 @@ +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_restart_lib.py + +DESCRIPTION + Library functions required to control the starting or restarting of + NEMO ocean model +''' +import glob +import os +import re +import sys +import time + +import common +import error +# +# SECTION ONE: +# +# Section one contains functions that can be called directly from other +# modules +# + +def setup_previous_restart_nl( + common_env, nemo_rst, ice_rst, top_rst, latest_nemo_dump): + ''' + Setup the path to the previous restart namelist if required + ''' + if common_env['CONTINUE'] == 'false': + sys.stdout.write('[INFO] New nemo run\n') + _setup_previous_restart_nl_nrun(nemo_rst, ice_rst, top_rst) + # The restart is in the current working directory in this case + return '.' + if os.path.isfile(latest_nemo_dump): + sys.stdout.write('[INFO] Restart data available in NEMO restart ' + 'directory %s. Restarting from previous task output\n' + '[INFO] Sourcing namelist file from the work ' + 'directory of the previous cycle\n' + % nemo_rst) + # find the previous work directory if there is one + return _setup_previous_restart_nl_crun(common_env) + sys.stderr.write('[FAIL] No restart data available in NEMO restart ' + 'directory:\n %s\n' % nemo_rst) + sys.exit(error.MISSING_MODEL_FILE_ERROR) + +def verify_fix_rst(restartdate, cyclepoint, nemo_rst): + ''' + Verify that the restart file for nemo is at the cyclepoint for the + start of this cycle. The cyclepoint variable has form + yyyymmddThhmmZ, restart date yyyymmdd. If they don't match, then + make sure that nemo restarts from the correct restart date + ''' + cycle_date_string = cyclepoint.split('T')[0] + if restartdate == cycle_date_string: + sys.stdout.write('[INFO] Validated NEMO restart date\n') + else: + # Write the message to both standard out and standard error + msg = '[WARN] The NEMO restart data does not match the ' \ + ' current cycle time\n.' \ + ' Cycle time is %s\n' \ + ' NEMO restart time is %s\n' \ + '[WARN] Automatically removing NEMO dumps ahead of ' \ + 'the current cycletime, and pick up the dump at ' \ + 'this time\n' % (cycle_date_string, restartdate) + sys.stdout.write(msg) + sys.stderr.write(msg) + #Remove all nemo restart files that are later than the correct + #cycle times + #Make our generic restart regular expression, to cover normal NEMO + #restart, and potential iceberg or passive tracer restart files, for + #both the rebuilt and non rebuilt cases + generic_rst_regex = r'(icebergs)?.*restart(_trc)?(_\d+)?\.nc' + all_restart_files = [f for f in os.listdir(nemo_rst) if + re.findall(generic_rst_regex, f)] + for restart_file in all_restart_files: + fname_date = re.findall(r'\d{8}', restart_file)[0] + if fname_date > cycle_date_string: + common.remove_file(os.path.join(nemo_rst, restart_file)) + restartdate = cycle_date_string + return restartdate + +def compile_nemo_restart_files(nemo_rst): + ''' + Compile a list of the NEMO restart files, if any exist. + We look for files conforming to the naming convention: + _yyyymmdd_restart_.nc where + may itself contain underscores, hence we + do not parse details based on counting the number of underscores. + ''' + nemo_restart_files = [f for f in os.listdir(nemo_rst) if + re.findall(r'.+_\d{8}_restart(_\d+)?\.nc', f)] + nemo_restart_files.sort() + if nemo_restart_files: + latest_nemo_dump = os.path.join(nemo_rst, nemo_restart_files[1]) + else: + latest_nemo_dump = 'unset' + # We need to ensure any lingering NEMO or iceberg retarts from + # previous runs are removed to ensure they're not accidentally + # picked up if we're starting from climatology on this occasion. + common.remove_file('restart.nc') + common.remove_file('restart_icebergs.nc') + common.remove_file('restart_trc.nc') + + if os.path.isfile(latest_nemo_dump): + nemo_init_dir = nemo_rst + else: + nemo_init_dir = '.' + return nemo_restart_files, latest_nemo_dump, nemo_init_dir + +def create_restart_direcs(restart_direcs, common_env): + ''' + Take a list of restart directorys. For an NRUN archive any existing + and create new ones + ''' + if common_env['CONTINUE'] == 'false': + # We only need to manipulate restart directories for NRUNS + # Create the time string here so that all directories will have the + # same time + time_str = time.strftime("%Y%m%d%H%M") + for direc in restart_direcs: + isdir = os.path.isdir(direc) + if isdir and (direc not in ('./', '.')): + sys.stdout.write('[INFO] directory is %s\n' % direc) + sys.stdout.write('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n') + os.rename(direc, '%s.%s' % (direc, time_str)) + os.makedirs(direc) + elif not isdir: + sys.stdout.write('[INFO] Creating NEMO restart directory:\n' + ' %s\n' % direc) + os.makedirs(direc) + + +# +# SECTION TWO: +# +# Section two contains functions that are private to this module +# + +def _setup_previous_restart_nl_nrun(nemo_rst, ice_rst, top_rst): + ''' + Delete the restarts when starting an NRUN + ''' + restart_groups = [nemo_rst+'/*restart*', nemo_rst+'/*trajectory*'] + if ice_rst: + restart_groups.append(ice_rst+'/*restart*') + if top_rst: + restart_groups.append(top_rst+'/*restart*') + for glob_exp in restart_groups: + for file_path in glob.glob(glob_exp): + common.remove_file(file_path) + +def _setup_previous_restart_nl_crun(common_env): + ''' + Setup directories and path to nemo namelist for CRUNS + ''' + if common_env['CONTINUE_FROM_FAIL'] == 'false': + if common_env['CNWP_SUB_CYCLING'] == 'True': + prev_workdir = common.find_previous_workdir(\ + common_env['CYLC_TASK_CYCLE_POINT'], + common_env['CYLC_TASK_WORK_DIR'], + common_env['CYLC_TASK_NAME'], + common_env['CYLC_TASK_PARAM_run']) + else: + prev_workdir = common.find_previous_workdir( \ + common_env['CYLC_TASK_CYCLE_POINT'], + common_env['CYLC_TASK_WORK_DIR'], + common_env['CYLC_TASK_NAME']) + return prev_workdir + return '' From e927f7ec217663342110c420d3e354095e878644 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 09:49:52 +0100 Subject: [PATCH 24/47] Add nemo runtime namcouple Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/nemo_runtime_namcouple.py | 112 ++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 Coupled_Drivers/nemo_runtime_namcouple.py diff --git a/Coupled_Drivers/nemo_runtime_namcouple.py b/Coupled_Drivers/nemo_runtime_namcouple.py new file mode 100644 index 00000000..a953c455 --- /dev/null +++ b/Coupled_Drivers/nemo_runtime_namcouple.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_runtime_namcouple.py + +DESCRIPTION + Library of function required for the creation of the NEMO namcouple + components at runtime +''' +import os +import sys +import write_namcouple +import error +try: + import f90nml +except ImportError: + pass + +def get_ocean_resol(nemo_nl_file, run_info): + ''' + Determine the ocean resolution. + This function is only used when creating the namcouple at run time. + ''' + + # Read in the resolution of ocean (existent of namelist_cfg has + # already been checked) + ocean_nml = f90nml.read(nemo_nl_file) + + # Check the required entries exist + if 'namcfg' not in ocean_nml: + sys.stderr.write('[FAIL] namcfg not found in namelist_cfg\n') + sys.exit(error.MISSING_OCN_RESOL_NML) + if 'jpiglo' not in ocean_nml['namcfg'] or \ + 'jpjglo' not in ocean_nml['namcfg'] or \ + 'cp_cfg' not in ocean_nml['namcfg'] or \ + 'jp_cfg' not in ocean_nml['namcfg']: + sys.stderr.write('[FAIL] cp_cfg, jp_cfg, jpiglo or jpjglo are ' + 'missing from namelist namcf in namelist_cfg\n') + sys.exit(error.MISSING_OCN_RESOL) + + # Check it is on orca grid + if ocean_nml['namcfg']['cp_cfg'] != 'orca': + sys.stderr.write('[FAIL] we can currently only handle the ' + 'ORCA grid\n') + sys.exit(error.NOT_ORCA_GRID) + + # Check this is a grid we recognise + if ocean_nml['namcfg']['jp_cfg'] == 25: + run_info['OCN_grid'] = 'orca025' + else: + run_info['OCN_grid'] = 'orca' + str(ocean_nml['namcfg']['jp_cfg']) + + # Store the ocean resolution + run_info['OCN_resol'] = [ocean_nml['namcfg']['jpiglo'], + ocean_nml['namcfg']['jpjglo']] + + return run_info + +def sent_coupling_fields(nemo_envar, run_info): + ''' + Write the coupling fields sent from NEMO into model_snd_list. + This function is only used when creating the namcouple at run time. + ''' + # Check that file specifying the coupling fields sent from + # NEMO is present + if not os.path.exists('OASIS_OCN_SEND'): + sys.stderr.write('[FAIL] OASIS_OCN_SEND is missing.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND) + + # Add toyatm to our list of executables + if not 'exec_list' in run_info: + run_info['exec_list'] = [] + run_info['exec_list'].append('toyoce') + + # Determine the ocean resolution + run_info = get_ocean_resol(nemo_envar['NEMO_NL'], run_info) + + # If using the default coupling option, we'll need to read the + # NEMO namelist later + run_info['nemo_nl'] = nemo_envar['NEMO_NL'] + + # Read the namelist + oasis_nml = f90nml.read('OASIS_OCN_SEND') + + # Check we have the expected information + if 'oasis_ocn_send_nml' not in oasis_nml: + sys.stderr.write('[FAIL] namelist oasis_ocn_send_nml is ' + 'missing from OASIS_OCN_SEND.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND_NML) + if 'oasis_ocn_send' not in oasis_nml['oasis_ocn_send_nml']: + sys.stderr.write('[FAIL] entry oasis_ocn_send is missing ' + 'from namelist oasis_ocn_send_nml in ' + 'OASIS_OCN_SEND.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND) + + # Create a list of fields sent from OCN + model_snd_list = \ + write_namcouple.add_to_cpl_list( \ + 'OCN', False, 0, + oasis_nml['oasis_ocn_send_nml']['oasis_ocn_send']) + + return run_info, model_snd_list From c62272ab0991bdc741582daa969160cc941acc0d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 09:55:20 +0100 Subject: [PATCH 25/47] Add ocn lib Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/ocn_lib.py | 211 +++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 Coupled_Drivers/ocn_lib.py diff --git a/Coupled_Drivers/ocn_lib.py b/Coupled_Drivers/ocn_lib.py new file mode 100644 index 00000000..c44cc6fe --- /dev/null +++ b/Coupled_Drivers/ocn_lib.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + ocn_lib.py + +DESCRIPTION + Library of function required for the ocean models +''' +from collections import namedtuple +import glob +import os +import re +import sys + +import common +import error + + +class ReadOcnNamelist: + ''' + A class to define the behaviours for reading from a wet model namelist. + We define the potential variables and their regular expressions to read + from the namelist file + ''' + def __init__(self, namelist_file): + ''' + Constructor for the object to read from given namelist file + ''' + self._to_read = {} + self.namelist_file = namelist_file + NamelistVal = namedtuple('NamelistVal', 'regex value') + self._variables \ + = {'nemo_first_step': NamelistVal(r'nn_it000=(.+),', None), + 'nemo_last_step': NamelistVal(r'nn_itend=(.+),', None), + 'nemo_step_int': NamelistVal(r'rn_rdt=(\d*)', None), + 'nemo_rst_date': NamelistVal(r'ln_rstdate=(.+),', None), + 'nemo_rst': NamelistVal( + r'cn_ocerst_outdir=[\"\'](.*?)[\"\']', None), + 'ice_rst': NamelistVal( + r'cn_icerst_outdir=[\"\'](.*?)[\"\']', None), + 'ln_icebergs': NamelistVal(r'ln_icebergs=(.+),', None), + 'top_rst': NamelistVal( + r'cn_trcrst_outdir=[\"\'](.*?)[\"\']', None)} + + def read_variables(self, *requested_variables): + ''' + Read an arbitary number of variable names and return their values. + If the variable doesnt exist, return none + ''' + # Reset _to_read + self._to_read = {} + for requested_var in requested_variables: + if requested_var not in self._variables.keys(): + sys.stderr.write('[FAIL] Unable to determine how to read the' + ' variable %s from file %s\n' % + (requested_var, self.namelist_file)) + sys.exit(error.MISSING_FILE_CONTENTS_ERROR) + self._to_read[requested_var] = self._variables[requested_var] + self._read_file() + # loop again to get the output in the correct order + return_list = [] + for requested_var in requested_variables: + return_list.append(self._to_read[requested_var].value) + if len(return_list) == 1: + return return_list[0] + return tuple(return_list) + + def _read_file(self): + '''Read the variables from the namelist file''' + with open(self.namelist_file, 'r') as nl_fh: + for line in nl_fh.readlines(): + for var in self._to_read.keys(): + try: + self._to_read[var] \ + = self._to_read[var]._replace(value=(re.findall( + self._to_read[var].regex, line)[0])) + except IndexError: + pass + + + +# +# Functions to interface to the library +# + +def setup_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, model_dscr): + ''' + Setup wet model restart file links (NEMO, TOP, SI3) + ''' + _create_restart_link_dir(restart_link_dir) + nproc_str_len = _get_nemo_nproc_str(init_dir) + if nproc_str_len > 0: + _setup_multiple_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, nproc_str_len) + else: + # It is likely we might have a rebuilt restart file + _setup_single_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, model_dscr) + + +def setup_nrun(start_envar, restart_link_dir, restart_link_label, init_dir, + warning_message): + ''' + Setup restart files for an NRUN + ''' + _create_restart_link_dir(restart_link_dir) + if start_envar: + len_proc_num = _get_nemo_nproc_str(init_dir) + if os.path.isfile(start_envar): + restart_link_label = '%s.nc' % restart_link_label + os.symlink(start_envar, + os.path.join(restart_link_dir, restart_link_label)) + ln_restart = ".true." + elif os.path.isfile('%s_%s.nc' % + (start_envar, '0'*len_proc_num)): + for fname in glob.glob('%s_%s.nc' % + (start_envar, '?'*len_proc_num)): + proc_number = fname.split('.')[-2][-len_proc_num:] + restart_link_file = os.path.join( + restart_link_dir, '%s_%s.nc' % (restart_link_label, + proc_number)) + common.remove_file(restart_link_file) + os.symlink(fname, restart_link_file) + ln_restart = ".true." + else: + sys.stderr.write('[FAIL] file %s not found\n' % (start_envar)) + sys.exit(error.MISSING_MODEL_FILE_ERROR) + else: + # Start envar is unset + sys.stdout.write('[WARN] %s\n' % warning_message) + ln_restart = ".false." + return ln_restart + +# +# Functions to manipulate restart file links +# + +def _get_nemo_nproc_str(restart_file_dir='.'): + ''' + Get the length of the string in the nemo restart file name containing + the processor number for unbuilt restart files. For example + restart_0012.nc would return 4. Takes in a directory to search for + the restart file. If no directory specified we assume it is in the + current directory + ''' + # Find how large the processor tag is on the restart files + # Get the zero tagged NEMO restart file + rst_file_glob = '%s/*_[0-9]*_restart_*0.nc' % restart_file_dir + try: + zero_nemo_rst = glob.glob(rst_file_glob)[0] + print(zero_nemo_rst) + # Get the length of the strings of zeros + len_proc_num = len(re.search(r'.*_(\d+).nc', zero_nemo_rst).group(1)) + except IndexError: + len_proc_num = 0 + return len_proc_num + + +def _setup_multiple_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, nproc_str_len): + ''' + Setup links for multiple (unrebuilt) restart files + ''' + for i_proc in range(nemo_nproc): + tag = str(i_proc).zfill(nproc_str_len) + rst_source = '%s/%s_%s.nc' % (init_dir, + file_name_label, + tag) + rst_link = os.path.join(restart_link_dir, + '%s_%s.nc' % (restart_link_label, tag)) + common.remove_file(rst_link) + + if os.path.isfile(rst_source): + os.symlink(rst_source, rst_link) + +def _setup_single_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, model_dscr): + ''' + Setup links for single (rebuilt) restart file + ''' + sys.stdout.write('[INFO] No %s sub-PE restarts found\n' % model_dscr) + rst_source = '%s/%s.nc' % (init_dir, file_name_label) + rst_link = os.path.join(restart_link_dir, + '%s.nc' % restart_link_label) + common.remove_file(rst_link) + if os.path.isfile(rst_source): + sys.stdout.write('[INFO] Using rebuilt %s restart file %s\n' + % (model_dscr, rst_source)) + os.symlink(rst_source, rst_link) + +def _create_restart_link_dir(dirname): + ''' + Try to create the restart link directory, if it already exists we carry + on as normal''' + if dirname != '.': + try: + os.mkdir(dirname) + except FileExistsError: + sys.stdout.write('[INFO] Restart link subdirectory %s exists\n' % + dirname) From 58f8c921bed831a831f08ecf0161d1bd301c497d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 11:13:41 +0100 Subject: [PATCH 26/47] Add path function testing to common unit tests --- Coupled_Drivers/unittests/test_common.py | 33 ++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Coupled_Drivers/unittests/test_common.py b/Coupled_Drivers/unittests/test_common.py index 769ab23c..c7732a6e 100644 --- a/Coupled_Drivers/unittests/test_common.py +++ b/Coupled_Drivers/unittests/test_common.py @@ -110,3 +110,36 @@ def test_remove_latest_hist_dir(self): glob.glob(self.dirpattern), expected_dirs ) + +class TestPathFunctions(unittest.TestCase): + ''' + Test the functions relating to dealing with file paths + ''' + def test_remove_trailing_slash_no_slash(self): + '''Test that with no trailing slash the path string gets passed + straight through''' + path = '/path/with/no/slash' + self.assertEqual(common.remove_trailing_slash(path), path) + + def test_remove_trailing_slash_with_slash(self): + '''Test that we successfully strip the trailing slash''' + path = '/path/with/trailing/slash/' + self.assertEqual(common.remove_trailing_slash(path), path[:-1]) + + def test_remove_three_trailing_slashes(self): + '''Test that we successfully remove three trailing slashes and + no initial forward slash''' + path = 'path/with/three/trailing/slashes///' + self.assertEqual(common.remove_trailing_slash(path), path[:-3]) + + def test_remove_trailing_space(self): + '''Test that we successfully remove trailing whitespacec''' + path = '/path/with/trailing/space ' + self.assertEqual(common.remove_trailing_slash(path), path[:-1]) + + def test_remove_combo(self): + '''Test removal of combination of whitespace and slashes''' + path = '/path/with/trailing/combo/ /\t ///' + expected_return = '/path/with/trailing/combo' + self.assertEqual(common.remove_trailing_slash(path), expected_return) + From 6da2a967ea573b014983c1e13f9cba2c9388e84a Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 11:34:58 +0100 Subject: [PATCH 27/47] Add nemo driver misc tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_nemo_driver_misc.py | 593 ++++++++++++++++++ 1 file changed, 593 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_driver_misc.py diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py new file mode 100644 index 00000000..ad46951d --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_driver_misc.py + +DESCRIPTION + Test miscellaneous functions in the Nemo Driver +''' + +import unittest +import unittest.mock as mock + +import os + +import error +import nemo_driver + + +class TestNemoDriverAssertions(unittest.TestCase): + ''' + Test the assertions for the NEMO drivers, and that when they fail + an assertion error is raised + ''' + def test_success_nemo_36(self): + '''If the nemo version is 3.6 and NEMO is in parallel mode, we run + correctly''' + nemo_envar = {'NEMO_VERSION': 306, + 'NEMO_NPROC': 10} + self.assertIsNone(nemo_driver._nemo_driver_assertions(nemo_envar)) + + def test_success_nemo_40(self): + '''If the nemo version is 4.0 and NEMO is in parallel mode, we run + correctly''' + nemo_envar = {'NEMO_VERSION': 400, + 'NEMO_NPROC': 2} + self.assertIsNone(nemo_driver._nemo_driver_assertions(nemo_envar)) + + def test_failure_nemo_serial(self): + '''If nemo is attempting to run in serial mode, assertion will fail''' + nemo_envar = {'NEMO_VERSION': 400, + 'NEMO_NPROC': 1} + with self.assertRaises(AssertionError) as context: + nemo_driver._nemo_driver_assertions(nemo_envar) + self.assertEqual('[FAIL] Nemo driver does not support the running' + ' of NEMO in serial mode\n', + str(context.exception)) + + def test_failure_nemo_34(self): + '''If nemo version is lower than 3.6 (ie 3.4) assertion will fail''' + nemo_envar = {'NEMO_VERSION': 304, + 'NEMO_NPROC': 2} + with self.assertRaises(AssertionError) as context: + nemo_driver._nemo_driver_assertions(nemo_envar) + self.assertEqual('[FAIL] The python drivers are only valid for NEMO' + ' versions 3.6 and later.\n', + str(context.exception)) + +class TestRunSubmodelCustom(unittest.TestCase): + ''' + Test the running of code that is custom to the submodels + ''' + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_si3_only( + self, mock_update_si3_nl, mock_update_top_nl): + common_env = {'models': 'si3'} + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', 'ln_restart', 'restart_ctl')) + mock_update_si3_nl.assert_called_once_with('nemo_envar') + mock_update_top_nl.assert_not_called() + + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_top_lnrestart_true( + self, mock_update_si3_nl, mock_update_top_nl): + '''Test the correct running when ln_restart is .true. and no si3''' + common_env = {'models': 'top'} + ln_restart = '.true.' + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', ln_restart, 'restart_ctl')) + mock_update_top_nl.assert_called_once_with( + 'nemo_envar', '.true.', 'restart_ctl', '.false.') + mock_update_si3_nl.assert_not_called() + + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_top_lnrestart_false_and_si3( + self, mock_update_si3_nl, mock_update_top_nl): + '''Test the correct running when ln_restart is .false. and si3''' + common_env = {'models': 'top si3'} + ln_restart = '.false.' + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', ln_restart, 'restart_ctl')) + mock_update_top_nl.assert_called_once_with( + 'nemo_envar', '.false.', 'restart_ctl', '.true.') + mock_update_si3_nl.assert_called_once_with('nemo_envar') + + @mock.patch('nemo_driver.sys.stderr.write') + def test_run_submodel_custom_top_lnrestart_invalid(self, mock_stderr): + '''Test that we exit with an error message if ln_restart has invalid + value''' + common_env = {'models': 'top'} + ln_restart = 'invalid' + with self.assertRaises(SystemExit) as context: + nemo_driver._run_submodel_custom(common_env, None, ln_restart, None) + self.assertEqual(context.exception.code, + error.INVALID_LOCAL_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] TOP: invalid ln_restart value: invalid\n') + + + +class TestProcessorRestartFiles(unittest.TestCase): + ''' + Test the call to set up the various restart files + ''' + @mock.patch('nemo_driver.ocn_lib.setup_restart') + def test_processor_restart_files_crun(self, mock_setup_rst): + '''Check the correct calls to setup_restart for CRUN''' + nemo_driver._processor_restart_files_crun( + 'nemo_init_dir', 'rst_link_dir', 'nemo_nproc', + 'nemo_dump_time', 'runid') + mock_setup_rst.assert_has_calls( + [mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_nemo_dump_time_restart', + 'restart', 'nemo_nproc', 'NEMO'), + mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_nemo_dump_time_restart_ice', + 'restart_ice', 'nemo_nproc', 'SI3'), + mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_icebergs_nemo_dump_time_restart', + 'restart_icebergs', 'nemo_nproc', 'Icebergs')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_nemo_only(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with no Ice or + TOP''' + mock_setup_nrun.side_effect = ['ln_restart', None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', + 'restart_icebergs', 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_with_si3(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with Ice but no + TOP''' + mock_setup_nrun.side_effect = ['ln_restart', None, None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'SI3_START': 'si3_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios si3'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', 'restart_icebergs', + 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n'), + mock.call( + 'si3_startdump', 'rst_link_dir', 'restart_ice', + 'nemo_init_dir', 'New SI3 run\n')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_with_top(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with TOP but no + ice''' + mock_setup_nrun.side_effect = ['ln_restart', None, None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'TOP_START': 'top_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios top'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', + 'restart_icebergs', 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n'), + mock.call( + 'top_startdump', 'rst_link_dir', 'restart_trc', + 'nemo_init_dir', 'New TOP run\n')]) + + +class TestCrunTimesteps(unittest.TestCase): + ''' + Test the setting up of timesteps for a CRUN + ''' + @mock.patch('nemo_driver.sys.stdout.write') + def test_integer_cycle_msg_continue_from_fail(self, mock_stdout): + '''Test an integer cycling when used with CONTINUE_FROM_FAIL true''' + common_env = {'CONTINUE_FROM_FAIL': 'true'} + expected_out = ('.true.', 2, 145) + rvalue = nemo_driver._crun_timesteps( + False, '144', 1200, None, common_env) + self.assertEqual(rvalue, expected_out) + mock_stdout.assert_called_once_with( + '[INFO] Nemo has previously completed 2 days\n') + + @mock.patch('nemo_driver.sys.stdout.write') + def test_date_cycle(self, mock_stdout): + '''Test an date cycling when used with CONTINUE_FROM_FAIL false''' + common_env = {'CONTINUE_FROM_FAIL': 'false'} + expected_out = ('.true.', 2, 73) + rvalue = nemo_driver._crun_timesteps( + True, None, None, 72, common_env) + self.assertEqual(rvalue, expected_out) + mock_stdout.assert_not_called() + +class TestNrunTimesteps(unittest.TestCase): + ''' + Test setting up of timesteps for an NRUN + ''' + def test_nrun_timesteps(self): + nemo_first_step = 72 + expected_return = (0, nemo_first_step, nemo_first_step - 1) + self.assertEqual( + expected_return, nemo_driver._nrun_timesteps(nemo_first_step)) + + +class TestWriteOutputFileToStdOut(unittest.TestCase): + ''' + Test the writing of an ocean output file to standard out + ''' + def setUp(self): + '''Create an output, one with unicode, one with errors to read''' + self.unicode_fn = 'unicode_output' + self.error_fn = 'error_output' + self.unicode_contents = '''A first line in the ascii range +A second line in the ascii range +A third line containing a unicode division in brackets (\u00F7) +A fourth line in the ascii range''' + self.error_contents = '''A first line with no error +A second line with E R R O R +A a third line with no error +A fourth line with E R R O R in the middle +A fifth line''' + with open(self.unicode_fn, 'w', encoding='utf8') as unicode_fh: + unicode_fh.write(self.unicode_contents) + with open(self.error_fn, 'w') as error_fh: + error_fh.write(self.error_contents) + + def tearDown(self): + '''Remove the test files''' + for test_f in [self.unicode_fn, self.error_fn]: + try: + os.remove(test_f) + except FileNotFoundError: + pass + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_error_zero(self, mock_stdout): + '''Test errors get written and we count two of them with no + error count provided''' + self.assertEqual( + 2, nemo_driver._write_output_file_to_stdout(self.error_fn)) + mock_stdout.assert_has_calls( + [mock.call( + '[INFO] Ocean output from file %s\n' % self.error_fn), + mock.call('A first line with no error\n'), + mock.call('A second line with E R R O R\n'), + mock.call('A a third line with no error\n'), + mock.call('A fourth line with E R R O R in the middle\n'), + mock.call('A fifth line')]) + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_error_non_zero(self, mock_stdout): + '''Test errors get written and we count four of them when input + error count is 2''' + self.assertEqual( + 4, nemo_driver._write_output_file_to_stdout(self.error_fn, 2)) + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_unicode_error_passed_in( + self, mock_stdout): + '''Test that the unicode string is written successfully without the + unicode character, and that with 2 errors passed in, 2 are passed out + as there are no E R R O Rs in this test''' + self.assertEqual( + 2, nemo_driver._write_output_file_to_stdout(self.unicode_fn, 2)) + mock_stdout.assert_has_calls( + [mock.call( + '[INFO] Ocean output from file %s\n' % self.unicode_fn), + mock.call('A first line in the ascii range\n'), + mock.call('A second line in the ascii range\n'), + mock.call('A third line containing a unicode division in' + ' brackets ()\n'), + mock.call('A fourth line in the ascii range')]) + +class TestWriteOceanToStdout(unittest.TestCase): + ''' + Test the listing through of output files and calling the function to write + them to standard out + ''' + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_succeed( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test for all potential restart files exisiting''' + mock_isfile.side_effect = [True, True, True, True] + mock_write_file.side_effect = ['error1', 'error2', 'error3', 'error4'] + self.assertEqual('error4', nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_has_calls( + [mock.call('ocean.output', 0), + mock.call('solver.stat', 'error1'), + mock.call('run.stat', 'error2'), + mock.call('icebergs.stat', 'error3')]) + mock_stdout.assert_not_called() + + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_failed( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test for all potential restart files missing''' + no_file_line = '[INFO] Nemo output file %s not avaliable\n' + mock_isfile.side_effect = [False, False, False, False] + self.assertEqual(0, nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_not_called() + mock_stdout.assert_has_calls( + [mock.call(no_file_line % 'ocean.output'), + mock.call(no_file_line % 'solver.stat'), + mock.call(no_file_line % 'run.stat'), + mock.call(no_file_line % 'icebergs.stat')]) + + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_nemo4_combo( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test the file combination for NEMO4 (no solver.stat)''' + no_file_line = '[INFO] Nemo output file %s not avaliable\n' + mock_isfile.side_effect = [True, False, True, True] + mock_write_file.side_effect = ['error1', 'error2', 'error3'] + self.assertEqual('error3', nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_has_calls( + [mock.call('ocean.output', 0), + mock.call('run.stat', 'error1'), + mock.call('icebergs.stat', 'error2')]) + mock_stdout.assert_called_once_with(no_file_line % 'solver.stat') + + +class TestMoveNLEndOfRun(unittest.TestCase): + ''' + Test the copying of the namelist files at the end of the run + ''' + def setUp(self): + '''Set up the environment variable object''' + self.nemo_envar_fin = {'NEMO_NL': 'nemo_namelist', + 'TOP_NL': 'top_namelist'} + + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_nemo_only(self, mock_copy, mock_isdir): + '''Test copying of namelist NEMO only''' + mock_isdir.return_value = True + nemo_driver._copy_nl_end_of_run(self.nemo_envar_fin, 'nemo_rst', None) + mock_isdir.assert_called_once_with('nemo_rst') + mock_copy.assert_called_once_with('nemo_namelist', 'nemo_rst') + + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_nemo_top(self, mock_copy, mock_isdir): + '''Test copying of namelist NEMO and TOP''' + mock_isdir.side_effect = [True, True] + nemo_driver._copy_nl_end_of_run( + self.nemo_envar_fin, 'nemo_rst', 'top_rst') + mock_isdir.assert_has_calls([mock.call('nemo_rst'), + mock.call('top_rst')]) + mock_copy.assert_has_calls([mock.call('nemo_namelist', 'nemo_rst'), + mock.call('top_namelist', 'top_rst')]) + + + + +class TestFinaliseExecutable(unittest.TestCase): + ''' + Test the finalise executable function + ''' + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.sys.stderr.write') + def test_finalize_executable_error_count( + self, mock_stderr, mock_write, mock_cwd, mock_stdout): + '''Test we exit if the error count is one''' + mock_write.return_value = 1 + mock_cwd.return_value = 'current_wd' + with self.assertRaises(SystemExit) as context: + nemo_driver._finalize_executable(None) + mock_stdout.assert_has_calls( + [mock.call('[INFO] finalizing NEMO\n'), + mock.call('[INFO] running finalize in current_wd\n')]) + mock_cwd.assert_called_once_with() + self.assertEqual(context.exception.code, + error.COMPONENT_MODEL_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] An error has been found with the NEMO run.' + ' Please investigate the ocean.output file for more' + ' details\n') + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver._copy_nl_end_of_run') + def test_finalize_executable_nemo_rst_top_rist( + self, mock_copy_nl_end_of_run, mock_read_nl, + mock_load_env, mock_write, mock_cwd, mock_stdout): + '''Test finalise when the nemo and top restarts are avaliable to move''' + common_env = {'models': 'model list'} + mock_write.return_value = 0 + mock_cwd.return_value = '' + mock_load_env.return_value = ({'NEMO_NL': 'nemo_namelist'}, + 'model list updated') + mock_read_nl.return_value = ('nemo_rst', None, None, 'top_rst') + nemo_driver._finalize_executable(common_env) + mock_stdout.assert_has_calls( + [mock.call('[INFO] finalizing NEMO\n'), + mock.call('[INFO] running finalize in \n')]) + mock_load_env.assert_called_once_with('final', 'model list') + mock_read_nl.assert_called_once_with(common_env, + {'NEMO_NL': 'nemo_namelist'}) + mock_copy_nl_end_of_run.assert_called_once_with( + {'NEMO_NL': 'nemo_namelist'}, 'nemo_rst', 'top_rst') + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_finalize_executable_nemo_rst_no_nemo_rst( + self, mock_copy, mock_isdir, mock_read_nl, + mock_load_env, mock_write, mock_cwd, mock_stdout): + '''Test finalise when there is no nemo_rst directory''' + mock_write.return_value = 0 + mock_cwd.return_value = '' + mock_load_env.return_value = ({'NEMO_NL': 'nemo_namelist'}, + 'model list updated') + mock_read_nl.return_value = ('nemo_rst', None, None, None) + mock_isdir.return_value = False + nemo_driver._finalize_executable({'models': 'model list updated'}) + mock_copy.assert_not_called() + +class TestRunDriver(unittest.TestCase): + ''' + Test the interface to run the driver + ''' + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_finalize(self, mock_write, mock_finalize, mock_setup): + '''Test finalise mode''' + rvalue = nemo_driver.run_driver('common_env', 'finalize', 'run_info') + self.assertEqual(rvalue, (None, None, 'run_info', None)) + mock_setup.assert_not_called() + mock_write.assert_not_called() + mock_finalize.assert_called_once_with('common_env') + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_failure(self, mock_write, mock_finalize, mock_setup): + '''Test finalise mode''' + rvalue = nemo_driver.run_driver('common_env', 'failure', 'run_info') + self.assertEqual(rvalue, (None, None, 'run_info', None)) + mock_setup.assert_not_called() + mock_finalize.assert_not_called() + mock_write.assert_called_once_with() + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._set_launcher_command') + @mock.patch('nemo_driver.nemo_runtime_namcouple.sent_coupling_fields') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_l_namcouple(self, mock_write, mock_finalize, mock_namc, + mock_launcher_cmd, mock_setup): + '''Test run mode with l_namcouple set in run info''' + run_info = {'l_namcouple': True} + common_env = {'ROSE_LAUNCHER': 'launcher'} + mock_setup.return_value = 'exe_envar' + mock_launcher_cmd.return_value = 'launch_cmd' + rvalue = nemo_driver.run_driver(common_env, 'run_driver', run_info) + self.assertEqual(rvalue, ('exe_envar', 'launch_cmd', run_info, None)) + mock_setup.assert_called_once_with(common_env) + mock_launcher_cmd.assert_called_once_with('launcher', 'exe_envar') + mock_namc.assert_not_called() + mock_finalize.assert_not_called() + mock_write.assert_not_called() + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._set_launcher_command') + @mock.patch('nemo_driver.nemo_runtime_namcouple.sent_coupling_fields') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_no_l_namcouple( + self, mock_write, mock_finalize, mock_namc, mock_launcher_cmd, + mock_setup): + '''Test run mode with l_namcouple set to false in run info''' + run_info = {'l_namcouple': False} + common_env = {'ROSE_LAUNCHER': 'launcher'} + mock_setup.return_value = 'exe_envar' + mock_launcher_cmd.return_value = 'launch_cmd' + mock_namc.return_value = ('run_info', 'model_snd_list') + rvalue = nemo_driver.run_driver(common_env, 'run_driver', run_info) + self.assertEqual( + rvalue, ('exe_envar', 'launch_cmd', 'run_info', 'model_snd_list')) + mock_setup.assert_called_once_with(common_env) + mock_launcher_cmd.assert_called_once_with('launcher', 'exe_envar') + mock_namc.assert_called_once_with('exe_envar', run_info) + mock_finalize.assert_not_called() + mock_write.assert_not_called() + +class TestVerifyRestart(unittest.TestCase): + ''' + Test the verification of NEMO restart files + ''' + @mock.patch('nemo_driver.nemo_restart_lib.verify_fix_rst') + def test_verify_restart_no_validation(self, mock_verify_fix): + '''Test for DRIVERS_VERIFY_RST false, nothing is done''' + common_env = {'DRIVERS_VERIFY_RST': 'False'} + self.assertEqual( + 'nemo_dump_time', nemo_driver._verify_restart( + common_env, None, 'nemo_dump_time', None)) + mock_verify_fix.assert_not_called() + + @mock.patch('nemo_driver.nemo_restart_lib.verify_fix_rst') + def test_verify_restart(self, mock_verify_fix): + '''Test for DRIVERS_VERIFY_RST True, validation performed''' + common_env = {'DRIVERS_VERIFY_RST': 'True', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point'} + nemo_env = {'NEMO_NPROC': '25'} + mock_verify_fix.return_value = 'nemo_dump_time_rtn' + self.assertEqual( + 'nemo_dump_time_rtn', nemo_driver._verify_restart( + common_env, nemo_env, 'nemo_dump_time', 'nemo_rst')) + mock_verify_fix.assert_called_once_with( + 'nemo_dump_time', 'cycle_point', 'nemo_rst') + From 36a06a3ae2bdd9c261dfff0e5a7462da5a6e64a4 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 11:42:40 +0100 Subject: [PATCH 28/47] Add nemo driver setup exe tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_nemo_driver_setup_exe.py | 274 ++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py diff --git a/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py b/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py new file mode 100644 index 00000000..fc099c73 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_driver_setup_exe.py + +DESCRIPTION + Test the setup executable function in NEMO driver. This has been separated + out, the function is essentially a list of calls to other functions but + it requires a large degree of mocking. +''' + +import unittest +import unittest.mock as mock + +import nemo_driver + + +class TestSetupExecutable(unittest.TestCase): + ''' + Test the setup_executable function + ''' + def setUp(self): + '''Commonalities required for setup executable''' + self.nemo_envar = {'OCEAN_LINK': 'ocean_link', + 'OCEAN_EXEC': 'ocean_exec', + 'NEMO_NL': 'nemo_namelist', + 'NEMO_NPROC': '25', + 'RST_LINK_DIR': 'rst_link_dir'} + self.nemo_envar_si3 = {**self.nemo_envar, + **{'SI3_NL': 'ice_namelist'}} + self.common_env = {'RUNID': 'runid', + 'models': 'nemo xios'} + self.common_env_si3 = {'RUNID': 'runid', + 'models': 'nemo si3 xios'} + + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.common.remove_file') + @mock.patch('nemo_driver.os.symlink') + @mock.patch('nemo_driver.nemo_lib.setup_dates') + @mock.patch('nemo_driver._nemo_driver_assertions') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.nemo_restart_lib.create_restart_direcs') + @mock.patch('nemo_driver.nemo_restart_lib.compile_nemo_restart_files') + @mock.patch('nemo_driver.nemo_restart_lib.setup_previous_restart_nl') + @mock.patch('nemo_driver.os.path.join') + @mock.patch('nemo_driver.nemo_lib.read_history_nl') + # Beginning of IF os.path.isfile + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._processor_restart_files_nrun') + @mock.patch('nemo_driver._nrun_timesteps') + # End if + @mock.patch('nemo_driver.nemo_lib.setup_nemo_runlen') + @mock.patch('nemo_driver.update_nemo_nl.update_nl') + # Not called + @mock.patch('nemo_driver._verify_restart') + @mock.patch('nemo_driver._processor_restart_files_crun') + @mock.patch('nemo_driver._crun_timesteps') + def test_setup_executable_nrun_no_si3(self, + mock_crun_timesteps, + mock_proc_files_crun, + mock_verify_rst, + # Not called + mock_update_nl, + mock_setup_runlen, + # End if + mock_nrun_timesteps, + mock_proc_files_nrun, + mock_path_isfile, + # Beginning of IF os.path.isfile + mock_read_history_nl, + mock_path_join, + mock_setup_previous_nl, + mock_compile_restarts, + mock_create_restart_direcs, + mock_read_current_nl, + mock_assertions, + mock_setup_dates, + mock_symlink, + mock_remove_file, + mock_load_envar): + '''Test setting up of executable for an NRUN with no SI3''' + mock_load_envar.return_value = (self.nemo_envar, \ + self.common_env['models']) + # The list is model basis + mock_setup_dates.return_value = ( + 'nleapy', [2020, 12, 1, 0, 0, 0], 'run_start', 'run_length', + 'run_days') + mock_read_current_nl.return_value = ('nemo_rst', None, None, None) + mock_compile_restarts.return_value = ( + 'nemo_restart_files', 'latest_nemo_dump', 'nemo_init_dir') + mock_setup_previous_nl.return_value = 'restart_nl_path' + mock_path_join.return_value = 'history_nemo_nl' + mock_read_history_nl.return_value = ( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date_bool') + # Beginning of IF os.path.isfile + mock_path_isfile.return_value = False + mock_proc_files_nrun.return_value = 'ln_restart' + mock_nrun_timesteps.return_value = ( + 'restart_ctl', 'nemo_next_step', 'nemo_last_step') + # End if + mock_setup_runlen.return_value = 'nemo_final_step' + + self.assertEqual( + self.nemo_envar, nemo_driver._setup_executable(self.common_env)) + + mock_load_envar.assert_called_once_with('setup', 'nemo xios') + mock_remove_file.assert_called_once_with('ocean_link') + mock_symlink.assert_called_once_with('ocean_exec', 'ocean_link') + mock_read_current_nl.assert_called_once_with(self.common_env, + self.nemo_envar) + mock_create_restart_direcs.assert_called_once_with( + ['nemo_rst'], self.common_env) + mock_compile_restarts.assert_called_once_with('nemo_rst') + mock_setup_previous_nl.assert_called_once_with( + self.common_env, 'nemo_rst', None, None, 'latest_nemo_dump') + mock_path_join.assert_called_once_with('restart_nl_path', + 'nemo_namelist') + mock_read_history_nl.assert_called_once_with('history_nemo_nl') + # Beginning of IF os.path.isfile + mock_path_isfile.assert_called_once_with('latest_nemo_dump') + mock_proc_files_nrun.assert_called_once_with(self.nemo_envar, + self.common_env, + 'nemo_init_dir') + mock_nrun_timesteps.assert_called_once_with('nemo_first_step') + # End if + mock_setup_runlen.assert_called_once_with( + self.common_env, 'run_start', [2020, 12, 1, 0, 0, 0], + 'nemo_step_int', 'run_days', 'run_length', 'nemo_next_step', + 'nemo_last_step') + mock_update_nl.assert_called_once_with( + self.common_env, self.nemo_envar, 'ln_restart', 'restart_ctl', + 'nemo_next_step', 'nemo_final_step', '20201201', 'nleapy') + + # Not called + mock_verify_rst.assert_not_called() + mock_proc_files_crun.assert_not_called() + mock_crun_timesteps.assert_not_called() + + + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.common.remove_file') + @mock.patch('nemo_driver.os.symlink') + @mock.patch('nemo_driver.nemo_lib.setup_dates') + @mock.patch('nemo_driver._nemo_driver_assertions') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.nemo_restart_lib.create_restart_direcs') + @mock.patch('nemo_driver.nemo_restart_lib.compile_nemo_restart_files') + @mock.patch('nemo_driver.nemo_restart_lib.setup_previous_restart_nl') + @mock.patch('nemo_driver.os.path.join') + @mock.patch('nemo_driver.nemo_lib.read_history_nl') + # Beginning of IF os.path.isfile + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver.re.findall') + @mock.patch('nemo_driver._verify_restart') + @mock.patch('nemo_driver._processor_restart_files_crun') + @mock.patch('nemo_driver._crun_timesteps') + # End if + @mock.patch('nemo_driver.nemo_lib.setup_nemo_runlen') + @mock.patch('nemo_driver.update_nemo_nl.update_nl') + # Not called + @mock.patch('nemo_driver._processor_restart_files_nrun') + @mock.patch('nemo_driver._nrun_timesteps') + @mock.patch('nemo_driver._run_submodel_custom') + def test_setup_executable_crun_si3(self, + mock_run_submodel_cust, + mock_nrun_timesteps, + mock_proc_files_nrun, + # Not called + mock_update_nl, + mock_setup_runlen, + # End if + mock_crun_timesteps, + mock_proc_files_crun, + mock_verify_rst, + mock_re_findall, + mock_path_isfile, + # Beginning of IF os.path.isfile + mock_read_history_nl, + mock_path_join, + mock_setup_previous_nl, + mock_compile_restarts, + mock_create_restart_direcs, + mock_read_current_nl, + mock_assertions, + mock_setup_dates, + mock_symlink, + mock_remove_file, + mock_load_envar): + '''Test setting up of executable for a CRUN with si3''' + mock_load_envar.return_value = (self.nemo_envar_si3, + self.common_env_si3['models']) + # The list is model basis + mock_setup_dates.return_value = ( + 'nleapy', [2020, 12, 1, 0, 0, 0], 'run_start', 'run_length', + 'run_days') + mock_read_current_nl.return_value = ( + 'nemo_rst', None, 'ice_rst', None) + mock_compile_restarts.return_value = ( + 'nemo_restart_files', 'latest_nemo_dump', 'nemo_init_dir') + mock_setup_previous_nl.return_value = 'restart_nl_path' + mock_path_join.return_value = 'history_nemo_nl' + mock_read_history_nl.return_value = ( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date_bool') + # Beginning of IF os.path.isfile + mock_path_isfile.return_value = True + mock_re_findall.return_value = ['nemo_dump_time_re'] + mock_verify_rst.return_value = 'nemo_dump_time_verify' + mock_crun_timesteps.return_value = ( + 'ln_restart', 'restart_ctl', 'nemo_next_step') + # End if + mock_setup_runlen.return_value = 'nemo_final_step' + + self.assertEqual( + self.nemo_envar_si3, + nemo_driver._setup_executable(self.common_env_si3)) + + mock_load_envar.assert_called_once_with('setup', 'nemo si3 xios') + mock_remove_file.assert_has_calls([mock.call('ocean_link'), + mock.call('restart.nc'), + mock.call('restart_ice.nc')]) + mock_symlink.assert_called_once_with('ocean_exec', 'ocean_link') + mock_read_current_nl.assert_called_once_with(self.common_env_si3, + self.nemo_envar_si3) + mock_create_restart_direcs.assert_called_once_with( + ['nemo_rst', 'ice_rst'], self.common_env_si3) + mock_compile_restarts.assert_called_once_with('nemo_rst') + mock_setup_previous_nl.assert_called_once_with( + self.common_env_si3, 'nemo_rst', 'ice_rst', + None, 'latest_nemo_dump') + mock_path_join.assert_called_once_with('restart_nl_path', + 'nemo_namelist') + mock_read_history_nl.assert_called_once_with('history_nemo_nl') + # Beginning of IF os.path.isfile + mock_path_isfile.assert_called_once_with('latest_nemo_dump') + mock_re_findall.assert_called_once_with( + r'_(\d*)_restart', 'latest_nemo_dump') + mock_verify_rst.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'nemo_dump_time_re', + 'nemo_rst') + mock_proc_files_crun.assert_called_once_with( + 'nemo_init_dir', 'rst_link_dir', 25, 'nemo_dump_time_verify', + 'runid') + mock_crun_timesteps.assert_called_once_with( + 'nemo_rst_date_bool', 'nemo_dump_time_verify', 'nemo_step_int', + 'nemo_last_step', self.common_env_si3) + # End if + mock_setup_runlen.assert_called_once_with( + self.common_env_si3, 'run_start', [2020, 12, 1, 0, 0, 0], + 'nemo_step_int', 'run_days', 'run_length', 'nemo_next_step', + 'nemo_last_step') + mock_update_nl.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'ln_restart', + 'restart_ctl', 'nemo_next_step', 'nemo_final_step', '20201201', + 'nleapy') + mock_run_submodel_cust.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'ln_restart', + 'restart_ctl') + + # Not called + mock_proc_files_nrun.assert_not_called() + mock_nrun_timesteps.assert_not_called() From 9aed725cb84b13442bf4b2c152a6584b940d669c Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 11:48:24 +0100 Subject: [PATCH 29/47] Add nemo driver submod tests. Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_nemo_driver_submod.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_driver_submod.py diff --git a/Coupled_Drivers/unittests/test_nemo_driver_submod.py b/Coupled_Drivers/unittests/test_nemo_driver_submod.py new file mode 100644 index 00000000..a4115d36 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_submod.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' + +import unittest +import unittest.mock as mock + +import nemo_driver + +class TestRunSubmodelControllers(unittest.TestCase): + ''' + Test the code that runs the sub model controllers from the nemo drivers + ''' + def setUp(self): + '''Set up a mock controller object to test the run controller''' + class MockController: + '''A mock controller to test the calls to run controller''' + def __init__(self, submodel): + '''Inititalise with a string containing the submodel name''' + self.name = submodel + self.expected_arguments = () + def run_controller(self, *arguments): + '''A function to test the running of the controller''' + assert arguments == self.expected_arguments + def clear(self): + '''Clear expected_arguments''' + self.expected_arguments = () + self.top_controller = MockController('TOP') + self.si3_controller = MockController('SI3') + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.importlib.import_module') + def test_no_controllers(self, mock_import, mock_stdout): + '''Test the code runs correctly if no controllers are used''' + nemo_envar = {'L_OCN_PASS_TRC': 'False'} + common_envar = {'models': 'um nemo'} + nemo_driver._run_submodel_controllers(nemo_envar, common_envar, + None, None, None) + mock_stdout.assert_called_once_with( + '[INFO] nemo_driver: Passive tracer code not active.\n') + mock_import.assert_not_called() + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.importlib.import_module') + def test_top_only(self, mock_import, mock_stdout): + '''Test that a top only call works, with upper case T''' + nemo_envar = {'L_OCN_PASS_TRC': 'True', + 'NEMO_NPROC': '10'} + common_envar = {'models': 'um nemo', + 'RUNID': 'runid', + 'DRIVERS_VERIFY_RST': 'verify_rst'} + #expected_arguments_to_controller + expected_args = (common_envar, 'restart_ctl', 10, 'runid', + 'verify_rst', 'nemo_dump_time', 'mode') + self.top_controller.expected_arguments = expected_args + mock_import.return_value = self.top_controller + nemo_driver._run_submodel_controllers(nemo_envar, common_envar, + 'restart_ctl', 'nemo_dump_time', + 'mode') + mock_stdout.assert_has_calls( + [mock.call('[INFO] nemo_driver: Passive tracer code is active.\n'), + mock.call('[INFO] Calling top_controller in mode mode\n')]) + mock_import.assert_called_once_with('top_controller') + self.top_controller.clear() From eca05d0e0391c8e3434cb365742c78efc7b50b82 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 11:53:24 +0100 Subject: [PATCH 30/47] Add nemo lib tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/unittests/test_nemo_lib.py | 139 +++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_lib.py diff --git a/Coupled_Drivers/unittests/test_nemo_lib.py b/Coupled_Drivers/unittests/test_nemo_lib.py new file mode 100644 index 00000000..5987c6d2 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_lib.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_lib.py + +DESCRIPTION + Test the non 'namelist' functions in the NEMO library +''' + +import unittest +import unittest.mock as mock + +import dr_env_lib.nemo_def +import nemo_lib +import error + + +class TestSetupDates(unittest.TestCase): + ''' + Test the setting up of dates for NEMO model run + ''' + def setUp(self): + ''' + Set up common_env dictionary containing model basis, run start, + run length. Calendar type to be added later + ''' + self.common_env = {'MODELBASIS': '2020,12,12,0,0,0', + 'TASKSTART': '2021,01,02,0,0,0', + 'TASKLENGTH': '3,4,5,0,0,0'} + # return values + self.basisls = [2020, 12, 12, 0, 0, 0] + self.startls = [2021, 1, 2, 0, 0, 0] + self.lenls = [3, 4, 5, 0, 0, 0] + + @mock.patch('nemo_lib.sys.stderr.write') + def test_setup_dates_invalid_calendar(self, mock_stderr): + '''Test failure if calendar is invalid''' + common_env = {'CALENDAR': 'invalid_calendar'} + with self.assertRaises(SystemExit) as context: + nemo_lib.setup_dates(common_env) + self.assertEqual(context.exception.code, error.INVALID_EVAR_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Calendar type %s not recognised\n' % common_env['CALENDAR']) + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_gregorian(self, mock_inc_days): + '''Test correct behaviour for gregorian calendar''' + common_env = {**{'CALENDAR': 'gregorian'}, **self.common_env} + expected_return = (1, self.basisls, self.startls, self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, 'gregorian') + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_360d(self, mock_inc_days): + '''Test correct behaviour for 360 day calendar''' + common_env = {**{'CALENDAR': '360day'}, **self.common_env} + expected_return = (30, self.basisls, self.startls, + self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, '360') + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_365d(self, mock_inc_days): + '''Test correct behaviour for 365 day calendar''' + common_env = {**{'CALENDAR': '365day'}, **self.common_env} + expected_return = (0, self.basisls, self.startls, + self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, '365') + + + + +class TestSetupNemoRunlen(unittest.TestCase): + ''' + Test the setting up of NEMO runlength for climate and coupled nwp style + cycling + ''' + def test_setup_nemo_runlen_climate_cycling(self): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is false''' + common_env = {'CONTINUE_FROM_FAIL': 'false'} + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_step_int = 1200. + nemo_last_step = 2232 + self.assertEqual(2307, + nemo_lib.setup_nemo_runlen( + common_env, None, None, nemo_step_int, run_days, + run_length, None, nemo_last_step)) + + def test_setup_nemo_runlen_cnwp_cycling(self): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is true + and the warning is not called''' + common_env = {'CONTINUE_FROM_FAIL': 'true', + 'LAST_DUMP_HOURS': '2'} + run_start = [2020, 1, 1, 1, 0, 0] + model_basis = [2019, 11, 1, 0, 0, 0] + nemo_step_int = 1200. + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_next_step = 4402 + self.assertEqual(4470, + nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, None)) + + @mock.patch('nemo_lib.sys.stderr.write') + def test_setup_nemo_runlen_cnwp_cycling_failure(self, mock_stderr): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is true and the + warning is called''' + common_env = {'CONTINUE_FROM_FAIL': 'true', + 'LAST_DUMP_HOURS': '2'} + run_start = [2020, 1, 1, 1, 0, 0] + model_basis = [2019, 11, 1, 0, 0, 0] + nemo_step_int = 1200. + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_next_step = 4401 + with self.assertRaises(SystemExit) as context: + nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, None) + self.assertEqual(context.exception.code, error.RESTART_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Last NEMO restarts not at correct time\n' + ' Last completed timestep 4401\n' + ' Expected next step 4402\n' + ' Actual next step 4401\n') From eaf1591b1feea79b74750c217c9921a877fd9cf8 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:11:23 +0100 Subject: [PATCH 31/47] Add nemo lib nl tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/unittests/test_nemo_lib_nl.py | 404 ++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_lib_nl.py diff --git a/Coupled_Drivers/unittests/test_nemo_lib_nl.py b/Coupled_Drivers/unittests/test_nemo_lib_nl.py new file mode 100644 index 00000000..57bd422d --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_lib_nl.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_lib_nl.py + +DESCRIPTION + Test the 'namelist' functions in the NEMO library +''' + +import unittest +import unittest.mock as mock + +import dr_env_lib.nemo_def +import dr_env_lib.ocn_cont_def + +import error +import nemo_lib + + +class TestReadCurrentCycleNLPublic(unittest.TestCase): + ''' + Test the public interface function to read the current namelist cycles + ''' + def setUp(self): + '''Set up the nemo environment variable object''' + self.nemo_envar = {'NEMO_NL': 'nemo_namelist', + 'SI3_NL': 'ice_namelist', + 'TOP_NL': 'top_namelist'} + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_no_models(self, mock_top, mock_si3, mock_nemo): + '''Test that if no models are present we just return nones''' + common_env = {'models': 'no recognised models'} + self.assertEqual( + (None, None, None, None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_not_called() + mock_top.assert_not_called() + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_only(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo only''' + common_env = {'models': 'nemo xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', None, None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_top.assert_not_called() + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_top(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo and top''' + common_env = {'models': 'nemo top xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_top.return_value = ('top_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', None, 'top_rst'), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_top.assert_called_once_with('top_namelist') + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_si3(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo and si3''' + common_env = {'models': 'nemo si3 xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_si3.return_value = ('si3_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', 'si3_rst', None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_si3.assert_called_once_with('ice_namelist') + mock_top.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_si3_top(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo si3 and top''' + common_env = {'models': 'nemo top si3 xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_si3.return_value = ('si3_rst') + mock_top.return_value = ('top_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', 'si3_rst', 'top_rst'), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_si3.assert_called_once_with('ice_namelist') + mock_top.assert_called_once_with('top_namelist') + + +class TestCheckNemoNL(unittest.TestCase): + ''' + Test the existance of the Nemo namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + nemo_envar = {'NEMO_NL': 'namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_nemonl(nemo_envar)) + mock_isfile.assert_called_once_with(nemo_envar['NEMO_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_nemonl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + nemo_envar = {'NEMO_NL': 'namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_nemonl(nemo_envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the nemo namelist file %s\n' % + nemo_envar['NEMO_NL']) + mock_isfile.assert_called_once_with(nemo_envar['NEMO_NL']) + + +class TestCheckSI3NL(unittest.TestCase): + ''' + Test the existance of the SI3 namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + envar = {'SI3_NL': 'ice_namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_si3nl(envar)) + mock_isfile.assert_called_once_with(envar['SI3_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_nemonl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + envar = {'SI3_NL': 'ice_namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_si3nl(envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the SI3 namelist file %s\n' % + envar['SI3_NL']) + mock_isfile.assert_called_once_with(envar['SI3_NL']) + +class TestCheckTopNL(unittest.TestCase): + ''' + Test the existance of the Top namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + envar = {'TOP_NL': 'top_namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_topnl(envar)) + mock_isfile.assert_called_once_with(envar['TOP_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_topnl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + envar = {'TOP_NL': 'top_namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_topnl(envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the TOP namelist file %s\n' % + envar['TOP_NL']) + mock_isfile.assert_called_once_with(envar['TOP_NL']) + +class TestReadCurrentCycleNLPrivate(unittest.TestCase): + ''' + Test we correctly read from the current cycle namelist. These are the + private functions that carry this out + ''' + def setUp(self): + '''Set up the unit test''' + class DummyReadOcnNL: + '''Dummy for ocn_lib.ReadOcnNamelist''' + def __init__(self, *vars_to_set_to_none): + '''Constructor containing any variables to set to none''' + self.vars_to_set_to_none = vars_to_set_to_none + def read_variables(self, *variables): + '''Check we pass through the correct variables in order, + setting desired ones to None''' + vars_to_return = [] + for var in variables: + vars_to_return.append( + None if var in self.vars_to_set_to_none else var) + return tuple(vars_to_return) + self.DummyNL = DummyReadOcnNL + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + @mock.patch('nemo_lib.string_to_boolean') + def test_read_current_cycle_nl_nemo( + self, mock_str_to_bool, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading NEMO restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'nemo_rst_rmslash' + mock_str_to_bool.return_value = 'bool_icebergs' + self.assertEqual( + ('nemo_rst_rmslash', 'bool_icebergs'), + nemo_lib._read_current_cycle_nl_nemo('namelist_path')) + mock_rmslash.assert_called_once_with('nemo_rst') + mock_str_to_bool.assert_called_once_with('ln_icebergs') + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + def test_read_current_cycle_nl_si3(self, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading SI3 restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'ice_rst_rmslash' + self.assertEqual( + 'ice_rst_rmslash', + nemo_lib._read_current_cycle_nl_si3('namelist_path')) + mock_rmslash.assert_called_once_with(('ice_rst',)) + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + def test_read_current_cycle_nl_top(self, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading TOP restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'top_rst_rmslash' + self.assertEqual( + 'top_rst_rmslash', + nemo_lib._read_current_cycle_nl_top('namelist_path')) + mock_rmslash.assert_called_once_with(('top_rst',)) + + + +class TestReadHistoryNl(unittest.TestCase): + ''' + Test the reading of History namelist for nemo + ''' + def setUp(self): + '''Test class for reading of history nl''' + class DummyReadNL: + '''Dummy ocn_lib.ReadOcnNamelist to test interface''' + def __init__(self, nlfilename): + '''Dummy constructor''' + pass + def read_variables(self, *testvar): + '''Verify called with, and return some information''' + assert testvar == ('nemo_first_step', 'nemo_last_step', + 'nemo_step_int', 'nemo_rst_date') + return '1', 'nemo_last_step', '1200', 'nemo_rst_date' + self.namelist = DummyReadNL('namelist_file') + + def test_check_nemo_laststep_digit(self): + '''If nemo last step is a digit, return an integer''' + self.assertEqual(2000, nemo_lib._check_nemo_last_step('2000')) + + def test_check_nemo_laststep_notdigit(self): + '''If nemo last step is not a digit, return 0''' + self.assertEqual(0, nemo_lib._check_nemo_last_step('set_by_system')) + + def test_string_to_boolean_true(self): + '''If the rst_date string is true, return boolean True''' + self.assertTrue(nemo_lib.string_to_boolean('.true.')) + + def test_string_to_boolean_upper_true(self): + '''If the rst_date string is TRUE, return boolean True''' + self.assertTrue(nemo_lib.string_to_boolean('.TRUE.')) + + def test_string_to_boolean_false(self): + '''If the rst_date string is false, return boolean False''' + self.assertFalse(nemo_lib.string_to_boolean('.false.')) + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.string_to_boolean') + @mock.patch('nemo_lib._check_nemo_last_step') + def test_read_history_nl(self, mock_last_step, mock_str_to_bool, + mock_read_nl): + '''Test the reading of history namelist, and conversions''' + expected_rvalue = (1, 'checked_last', 1200, 'converted_rst') + mock_read_nl.return_value = self.namelist + mock_last_step.return_value = 'checked_last' + mock_str_to_bool.return_value = 'converted_rst' + self.assertEqual( + expected_rvalue, nemo_lib.read_history_nl('namelist_file')) + mock_last_step.assert_called_once_with('nemo_last_step') + mock_str_to_bool.assert_called_once_with('nemo_rst_date') + + +class TestLoadEnvironmentVariables(unittest.TestCase): + ''' + Test the correct reading of environment variables for various submodels + ''' + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_setup_nemo_only(self, mock_load_def, mock_check_nemo, mock_load): + '''Test the correct loading of nemo setup''' + mock_load.return_value = 'nemo_envar' + mock_load_def.return_value = {'L_OCN_PASS_TRC': 'False'} + self.assertEqual( + ({'L_OCN_PASS_TRC': 'False'}, 'nemo xios'), + nemo_lib.load_environment_variables('setup', 'nemo xios')) + mock_load_def.assert_called_once_with( + 'nemo_envar', dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL) + mock_check_nemo.assert_called_once_with({'L_OCN_PASS_TRC': 'False'}) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_final_nemo_only(self, mock_load_def, mock_check_nemo, mock_load): + '''Test the correct loading of nemo final''' + mock_load.return_value = 'nemo_envar' + mock_load_def.return_value = {'L_OCN_PASS_TRC': 'False'} + self.assertEqual( + ({'L_OCN_PASS_TRC': 'False'}, 'nemo xios'), + nemo_lib.load_environment_variables('final', 'nemo xios')) + mock_load_def.assert_called_once_with( + 'nemo_envar', dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL) + mock_check_nemo.assert_called_once_with({'L_OCN_PASS_TRC': 'False'}) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('nemo_lib._check_si3nl') + @mock.patch('nemo_lib._check_topnl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_setup_nemo_top_si3( + self, mock_load_def, mock_check_top, mock_check_si3, + mock_check_nemo, mock_load): + '''Test the correct loading of setup for all 3 models''' + mock_load.return_value = 'nemo_envar' + nemo_load_dict = {'L_OCN_PASS_TRC': 'True'} + si3_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes'} + top_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes', + 'TOP_LOADED': 'yes'} + mock_load_def.side_effect = [nemo_load_dict, si3_load_dict, + top_load_dict] + self.assertEqual( + (top_load_dict, 'nemo xios si3 top'), + nemo_lib.load_environment_variables('setup', 'nemo xios si3')) + mock_load_def.assert_has_calls( + [mock.call('nemo_envar', + dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL), + mock.call(nemo_load_dict, + dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_INITIAL), + mock.call(si3_load_dict, + dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_INITIAL)]) + mock_check_nemo.assert_called_once_with(nemo_load_dict) + mock_check_si3.assert_called_once_with(si3_load_dict) + mock_check_top.assert_called_once_with(top_load_dict) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('nemo_lib._check_si3nl') + @mock.patch('nemo_lib._check_topnl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_final_nemo_top_si3( + self, mock_load_def, mock_check_top, mock_check_si3, + mock_check_nemo, mock_load): + '''Test the correct loading of final for all 3 models - Note that + si3 doesnt have any final environment variables''' + mock_load.return_value = 'nemo_envar' + nemo_load_dict = {'L_OCN_PASS_TRC': 'True'} + si3_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes'} + top_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes', + 'TOP_LOADED': 'yes'} + mock_load_def.side_effect = [nemo_load_dict, si3_load_dict, + top_load_dict] + self.assertEqual( + (top_load_dict, 'nemo xios si3 top'), + nemo_lib.load_environment_variables('final', 'nemo xios si3')) + mock_load_def.assert_has_calls( + [mock.call('nemo_envar', + dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL), + mock.call(nemo_load_dict, + dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_FINAL), + mock.call(si3_load_dict, + dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_FINAL)]) + mock_check_nemo.assert_called_once_with(nemo_load_dict) + mock_check_si3.assert_called_once_with(si3_load_dict) + mock_check_top.assert_called_once_with(top_load_dict) From 7cfdab48a0373dfb0f0beb1f22bfb96222f6b6dd Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:12:28 +0100 Subject: [PATCH 32/47] Add nemo restart lib nl tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_nemo_restart_lib_nl.py | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py diff --git a/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py b/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py new file mode 100644 index 00000000..b0fbf84d --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_restart_lib_nl.py + +DESCRIPTION + Test the 'namelist' functions in the NEMO restart library +''' + +import unittest +import unittest.mock as mock + +import error +import nemo_restart_lib + +class TestSetupPreviousRestart(unittest.TestCase): + ''' + Test the sourcing from previous work directories in NEMO library + ''' + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_new_run( + self, mock_isfile, mock_setup_crun, mock_setup_nrun, mock_stdout): + '''Test correct calls for NRUN''' + common_env = {'CONTINUE': 'false'} + rvalue = nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', None) + self.assertEqual(rvalue, '.') + mock_stdout.assert_called_once_with('[INFO] New nemo run\n') + mock_setup_nrun.assert_called_once_with( + 'nemo_rst', 'ice_rst', 'top_rst') + mock_setup_crun.assert_not_called() + mock_isfile.assert_not_called() + + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_cont_run( + self, mock_isfile, mock_setup_crun, mock_setup_nrun, mock_stdout): + '''Test correct calls for CRUN''' + common_env = {'CONTINUE': 'true'} + mock_isfile.return_value = True + mock_setup_crun.return_value = 'setup_crun' + rvalue = nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', 'nemo_dump') + self.assertEqual(rvalue, 'setup_crun') + mock_stdout.assert_called_once_with( + '[INFO] Restart data available in NEMO restart ' + 'directory nemo_rst. Restarting from previous task output\n' + '[INFO] Sourcing namelist file from the work ' + 'directory of the previous cycle\n') + mock_setup_nrun.assert_not_called() + mock_setup_crun.assert_called_once() + mock_isfile.assert_called_once_with('nemo_dump') + + @mock.patch('nemo_restart_lib.sys.stderr.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_fail(self, mock_isfile, mock_setup_crun, + mock_setup_nrun, mock_stderr): + '''Test the failure mode''' + common_env = {'CONTINUE': 'true'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as context: + nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', 'nemo_dump') + self.assertEqual(context.exception.code, error.MISSING_MODEL_FILE_ERROR) + mock_setup_nrun.assert_not_called() + mock_setup_crun.assert_not_called() + mock_isfile.assert_called_once_with('nemo_dump') + mock_stderr.assert_called_once_with( + '[FAIL] No restart data available in NEMO restart ' + 'directory:\n nemo_rst\n') + + @mock.patch('nemo_restart_lib.glob.glob') + @mock.patch('nemo_restart_lib.common.remove_file') + def test_setup_previous_restart_nl_nrun(self, mock_rmfile, mock_glob): + '''Test the removal for restarts for NRUNS so we dont acidentally + pick them up''' + mock_glob.side_effect = [['path1'], ['path2'], ['path3'], ['path4']] + nemo_restart_lib._setup_previous_restart_nl_nrun( + 'nemo_rst', 'ice_rst', 'top_rst') + mock_glob.assert_has_calls([mock.call('nemo_rst/*restart*'), + mock.call('nemo_rst/*trajectory*'), + mock.call('ice_rst/*restart*'), + mock.call('top_rst/*restart*')]) + mock_rmfile.assert_has_calls([mock.call('path1'), + mock.call('path2'), + mock.call('path3'), + mock.call('path4')]) + + @mock.patch('nemo_restart_lib.glob.glob') + @mock.patch('nemo_restart_lib.common.remove_file') + def test_setup_previous_restart_nl_nrun_no_ice( + self, mock_rmfile, mock_glob): + '''Test the removal for restarts for NRUNS so we dont acidentally + pick them up. In this case there is no ice restart set''' + mock_glob.side_effect = [['path1', 'path2'], ['path3']] + nemo_restart_lib._setup_previous_restart_nl_nrun( + 'nemo_rst', None, None) + mock_glob.assert_has_calls([mock.call('nemo_rst/*restart*'), + mock.call('nemo_rst/*trajectory*')]) + mock_rmfile.assert_has_calls([mock.call('path1'), + mock.call('path2'), + mock.call('path3')]) + + def test_setup_previous_restart_nl_crun_cont_from_fail(self): + '''If continue from fail, we run from the current work directory + and so the path returned will be a blank string''' + common_env = {'CONTINUE_FROM_FAIL': 'true'} + self.assertEqual( + '', nemo_restart_lib._setup_previous_restart_nl_crun(common_env)) + + @mock.patch('common.find_previous_workdir') + def test_setup_previous_restart_nl_crun_sub_cycle(self, mock_workdir): + '''Test the Coupled NWP sub cycling mode''' + common_env = {'CONTINUE_FROM_FAIL': 'false', + 'CNWP_SUB_CYCLING': 'True', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point', + 'CYLC_TASK_WORK_DIR': 'work_dir', + 'CYLC_TASK_NAME': 'task_name', + 'CYLC_TASK_PARAM_run': 'param_run'} + mock_workdir.return_value = 'prev_workdir' + self.assertEqual('prev_workdir', + nemo_restart_lib._setup_previous_restart_nl_crun( + common_env)) + mock_workdir.assert_called_once_with( + 'cycle_point', 'work_dir', 'task_name', 'param_run') + + @mock.patch('common.find_previous_workdir') + def test_setup_previous_restart_nl_crun_climate(self, mock_workdir): + '''Test the climate cycling mode''' + common_env = {'CONTINUE_FROM_FAIL': 'false', + 'CNWP_SUB_CYCLING': 'false', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point', + 'CYLC_TASK_WORK_DIR': 'work_dir', + 'CYLC_TASK_NAME': 'task_name'} + mock_workdir.return_value = 'prev_workdir' + self.assertEqual('prev_workdir', + nemo_restart_lib._setup_previous_restart_nl_crun( + common_env)) + mock_workdir.assert_called_once_with('cycle_point', 'work_dir', + 'task_name') From fc61f73ee89bde58502e650ec79296500199e1f7 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:17:37 +0100 Subject: [PATCH 33/47] Add nemo restart lib rst tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_nemo_restart_lib_rst.py | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py diff --git a/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py b/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py new file mode 100644 index 00000000..140c9c87 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_restart_lib_rst.py + +DESCRIPTION + Test the 'restart' functions in the NEMO restart library +''' +import unittest +import unittest.mock as mock + +import error +import nemo_restart_lib + +class TestVerifyFixRst(unittest.TestCase): + ''' + Test the verification that the nemo start dump date is consistent with the + start of the cycle + ''' + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_verify_fix_rst_does_match(self, mock_stdout): + '''Test a matching cycle point''' + restart_date = '20210423' + cyclepoint = '20210423T1150Z' + self.assertEqual( + restart_date, nemo_restart_lib.verify_fix_rst( + restart_date, cyclepoint, None)) + mock_stdout.assert_called_once_with( + '[INFO] Validated NEMO restart date\n') + + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib.sys.stderr.write') + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.join') + def test_verify_fix_rst_perform_fix( + self, mock_pathjoin, mock_rmfile, mock_findall, mock_lsdir, + mock_stderr, mock_stdout): + '''Test the removal of later restart files''' + expected_msg = '[WARN] The NEMO restart data does not match the ' \ + ' current cycle time\n.' \ + ' Cycle time is 20210423\n' \ + ' NEMO restart time is 20210424\n' \ + '[WARN] Automatically removing NEMO dumps ahead of ' \ + 'the current cycletime, and pick up the dump at ' \ + 'this time\n' + restart_date = '20210424' + cyclepoint = '20210423T1150Z' + restart_regex = r'(icebergs)?.*restart(_trc)?(_\d+)?\.nc' + date_regex = r'\d{8}' + mock_findall.side_effect = [True, False, True, True, + ['20210423'], ['20210422'], ['20210425']] + mock_lsdir.return_value = ['file1', 'file4', 'file2', 'file3'] + mock_pathjoin.return_value = 'joined_rst_path' + self.assertEqual( + '20210423', nemo_restart_lib.verify_fix_rst( + restart_date, cyclepoint, 'nemo_rst')) + mock_findall.assert_has_calls( + [mock.call(restart_regex, 'file1'), + mock.call(restart_regex, 'file4'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file3'), + mock.call(date_regex, 'file1'), + mock.call(date_regex, 'file2'), + mock.call(date_regex, 'file3')]) + mock_pathjoin.assert_called_once_with('nemo_rst', 'file3') + mock_rmfile.assert_called_once_with('joined_rst_path') + mock_stdout.assert_called_once_with(expected_msg) + mock_stderr.assert_called_once_with(expected_msg) + + +class TestCompileNemoRestartFiles(unittest.TestCase): + ''' + Test the determination of NEMO restart files + ''' + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.os.path.join') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_restart_files_latest_dump( + self, mock_isfile, mock_rmfile, mock_pathjoin, + mock_re_findall, mock_listdir): + '''Test behaviour when there are restart files and dump''' + restart_regex = r'.+_\d{8}_restart(_\d+)?\.nc' + # reverse order so they can later be sorted + mock_listdir.return_value = ['file3', 'file2', 'file1'] + mock_re_findall.side_effect = [True, False, True] + mock_pathjoin.return_value = 'latest_nemo_dump' + mock_isfile.return_value = True + self.assertEqual((['file1', 'file3'], 'latest_nemo_dump', 'nemo_rst'), + nemo_restart_lib.compile_nemo_restart_files( + 'nemo_rst')) + mock_listdir.assert_called_once_with('nemo_rst') + mock_re_findall.assert_has_calls([mock.call(restart_regex, 'file3'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file1')]) + mock_pathjoin.assert_called_once_with('nemo_rst', 'file3') + mock_rmfile.assert_has_calls([mock.call('restart.nc'), + mock.call('restart_icebergs.nc'), + mock.call('restart_trc.nc')]) + mock_isfile.assert_called_once_with('latest_nemo_dump') + + + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.os.path.join') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_no_restart_no_dump( + self, mock_isfile, mock_rmfile, mock_pathjoin, + mock_re_findall, mock_listdir): + '''Test behaviour when there are no restart files and no dumps''' + restart_regex = r'.+_\d{8}_restart(_\d+)?\.nc' + # reverse order so they can later be sorted + mock_listdir.return_value = ['file3', 'file2', 'file1'] + mock_re_findall.side_effect = [False, False, False] + mock_isfile.return_value = False + self.assertEqual(([], 'unset', '.'), + nemo_restart_lib.compile_nemo_restart_files( + 'nemo_rst')) + mock_listdir.assert_called_once_with('nemo_rst') + mock_re_findall.assert_has_calls([mock.call(restart_regex, 'file3'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file1')]) + mock_pathjoin.assert_not_called() + mock_rmfile.assert_has_calls([mock.call('restart.nc'), + mock.call('restart_icebergs.nc'), + mock.call('restart_trc.nc')]) + mock_isfile.assert_called_once_with('unset') + +class TestCreateRestartDirecs(unittest.TestCase): + ''' + Test the archiving and creation of NEMO restart directories + ''' + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + def test_crun(self, mock_mkdir, mock_rename): + '''For a CRUN we dont rename or make directories''' + common_env = {'CONTINUE': 'true'} + nemo_restart_lib.create_restart_direcs(None, common_env) + mock_mkdir.assert_not_called() + mock_rename.assert_not_called() + + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_false_options(self, mock_stdout, mock_mkdir, mock_rename, + mock_isdir): + '''Test the option for directly creating a restart dir, continue + is false, and the directory doesnt exist''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['NEMORESTART'] + mock_isdir.return_value = False + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_isdir.assert_called_once_with('NEMORESTART') + mock_mkdir.assert_called_once_with('NEMORESTART') + mock_rename.assert_not_called() + mock_stdout.assert_called_once_with( + '[INFO] Creating NEMO restart directory:\n NEMORESTART\n') + + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_cwd(self, mock_stdout, mock_mkdir, mock_rename, mock_isdir): + '''Test that if the directories are either . or ./ we dont do + anything''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['./', '.'] + mock_isdir.side_effect = [True, True] + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_isdir.assert_has_calls([mock.call('./'), mock.call('.')]) + mock_mkdir.assert_not_called() + mock_stdout.assert_not_called() + mock_rename.assert_not_called() + + @mock.patch('nemo_restart_lib.time.strftime') + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_multi(self, mock_stdout, mock_mkdir, mock_rename, mock_isdir, + mock_time): + '''Test if two restart directorys exist, and one doesnt, that the + two that exist get archived and new ones are created in their place''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['EXIST_1', 'EXIST_2', 'DONT_EXIST', '.'] + mock_isdir.side_effect = [True, True, False, True] + mock_time.return_value = '202104161924' + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_time.assert_called_once_with("%Y%m%d%H%M") + mock_rename.assert_has_calls([ + mock.call('EXIST_1', 'EXIST_1.202104161924'), + mock.call('EXIST_2', 'EXIST_2.202104161924')]) + mock_mkdir.assert_has_calls([mock.call('EXIST_1'), + mock.call('EXIST_2')]) + mock_stdout.assert_has_calls([ + mock.call('[INFO] directory is EXIST_1\n'), + mock.call('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n'), + mock.call('[INFO] directory is EXIST_2\n'), + mock.call('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n')]) From a78f6d98e0b1d3265ce4b90fe5e63b8fa21284c9 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:20:55 +0100 Subject: [PATCH 34/47] Add ocn lib tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/unittests/test_ocn_lib.py | 340 ++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_ocn_lib.py diff --git a/Coupled_Drivers/unittests/test_ocn_lib.py b/Coupled_Drivers/unittests/test_ocn_lib.py new file mode 100644 index 00000000..d5b6f560 --- /dev/null +++ b/Coupled_Drivers/unittests/test_ocn_lib.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' + +import unittest +import unittest.mock as mock + +from collections import namedtuple +import io +import os +import error +import ocn_lib + +class TestReadOcnNamelist(unittest.TestCase): + ''' + Test the reading of wet model namelists + ''' + def setUp(self): + '''Set up a dummy namelist file''' + self.nl_name = 'test_nl_file' + file_contents = ''' +&TESTNL +nn_it000=1, +nn_itend=2232, +ln_rstdate=.true., +rn_rdt=1200, +dummy=32, +/ + ''' + with open(self.nl_name, 'w') as f_h: + f_h.write(file_contents) + + def tearDown(self): + '''Delete the dummy NL file at the end of thes test''' + try: + os.remove(self.nl_name) + except FileNotFoundError: + pass + + def test_variables_database(self): + '''Test the dictionary containing variables and regular expressions + remains unchaged''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + NamelistVal = namedtuple('NamelistVal', 'regex value') + expected_variables \ + = {'nemo_first_step': NamelistVal(r'nn_it000=(.+),', None), + 'nemo_last_step': NamelistVal(r'nn_itend=(.+),', None), + 'nemo_step_int': NamelistVal(r'rn_rdt=(\d*)', None), + 'nemo_rst_date': NamelistVal(r'ln_rstdate=(.+),', None), + 'nemo_rst': NamelistVal( + r'cn_ocerst_outdir=[\"\'](.*?)[\"\']', None), + 'ice_rst': NamelistVal( + r'cn_icerst_outdir=[\"\'](.*?)[\"\']', None), + 'ln_icebergs': NamelistVal(r'ln_icebergs=(.+),', None), + 'top_rst': NamelistVal( + r'cn_trcrst_outdir=[\"\'](.*?)[\"\']', None)} + self.assertEqual(expected_variables, + read_ocn_namelist_object._variables) + del read_ocn_namelist_object + + def test_read_variables_multiple(self): + '''Test reading in of the variables from the namelist''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + expected_out = ('1', '2232', '.true.', '1200') + self.assertEqual( + expected_out, read_ocn_namelist_object.read_variables( + 'nemo_first_step', 'nemo_last_step', 'nemo_rst_date', + 'nemo_step_int')) + del read_ocn_namelist_object + + def test_read_variables_single(self): + '''Test that reading a single variable returns that variable and not + a tuple''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + expected_out = '1' + self.assertEqual(expected_out, + read_ocn_namelist_object.read_variables( + 'nemo_first_step')) + del read_ocn_namelist_object + + @mock.patch('ocn_lib.sys.stderr.write') + def test_read_variables_fail(self, mock_stderr): + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + with self.assertRaises(SystemExit) as context: + read_ocn_namelist_object.read_variables('invalid_var') + self.assertEqual(context.exception.code, + error.MISSING_FILE_CONTENTS_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Unable to determine how to read the variable invalid_var' + ' from file %s\n' % (self.nl_name)) + del read_ocn_namelist_object + + + + +class TestGetNemoNprocStr(unittest.TestCase): + ''' + Unit tests for determining the number of zeros in the unrebuilt NEMO + restart files + ''' + @mock.patch('ocn_lib.glob.glob') + def test_index_error_exception(self, mock_glob): + '''Test that if the index error is thrown, we have a returned length + of zero''' + mock_glob.side_effect = IndexError + self.assertEqual(ocn_lib._get_nemo_nproc_str(), 0) + + @mock.patch('ocn_lib.glob.glob') + def test_default_restart_directory(self, mock_glob): + '''Test correct behaviour with default dictionary''' + mock_glob.return_value = ['oce_2021_restart_000.nc'] + self.assertEqual(ocn_lib._get_nemo_nproc_str(), 3) + mock_glob.assert_called_with('./*_[0-9]*_restart_*0.nc') + + @mock.patch('ocn_lib.glob.glob') + def test_non_default_restart_directory(self, mock_glob): + '''Test correct behaviour with non default directory''' + mock_glob.return_value = ['oce_2021_restart_0.nc'] + self.assertEqual(ocn_lib._get_nemo_nproc_str('custom_dir'), 1) + mock_glob.assert_called_with('custom_dir/*_[0-9]*_restart_*0.nc') + + +class TestSetupMultipleRestart(unittest.TestCase): + ''' + Unit tests for setting up of multiple restart files + ''' + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_two_restarts(self, mock_slink, mock_isfile, mock_rmfile): + '''Test the correct linking of two restart files. Two nemo processors + but a nproc string length of 4''' + mock_isfile.side_effect = [True, True] + ocn_lib._setup_multiple_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 2, 4) + mock_rmfile.assert_has_calls([mock.call('rst_link_dir/lnname_0000.nc'), + mock.call('rst_link_dir/lnname_0001.nc')]) + mock_isfile.assert_has_calls([mock.call('init_dir/fname_0000.nc'), + mock.call('init_dir/fname_0001.nc')]) + mock_slink.assert_has_calls([mock.call('init_dir/fname_0000.nc', + 'rst_link_dir/lnname_0000.nc'), + mock.call('init_dir/fname_0001.nc', + 'rst_link_dir/lnname_0001.nc')]) + + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_no_file(self, mock_slink, mock_isfile, mock_rmfile): + '''Test that if a file isn't found it isn't linked''' + mock_isfile.return_value = False + ocn_lib._setup_multiple_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 1, 4) + mock_rmfile.assert_called_once_with('rst_link_dir/lnname_0000.nc') + mock_isfile.assert_called_once_with('init_dir/fname_0000.nc') + mock_slink.assert_not_called() + + +class TestSetupSingleRestart(unittest.TestCase): + ''' + Unit tests for setting up a single restart file + ''' + @mock.patch('ocn_lib.sys.stdout.write') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_single_rst(self, mock_slink, mock_isfile, mock_rmfile, + mock_stdout): + '''Test the correct linking of a single restart file''' + mock_isfile.return_value = True + ocn_lib._setup_single_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_rmfile.assert_called_once_with('rst_link_dir/lnname.nc') + mock_isfile.assert_called_once_with('init_dir/fname.nc') + mock_slink.assert_called_once_with('init_dir/fname.nc', + 'rst_link_dir/lnname.nc') + mock_stdout.assert_has_calls( + [mock.call('[INFO] No MODEL sub-PE restarts found\n'), + mock.call('[INFO] Using rebuilt MODEL restart file' + ' init_dir/fname.nc\n')]) + + @mock.patch('ocn_lib.sys.stdout.write') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_single_rst_no_file(self, mock_slink, mock_isfile, mock_rmfile, + mock_stdout): + '''Test that if the file is not found, no link attempted''' + mock_isfile.return_value = False + ocn_lib._setup_single_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_rmfile.assert_called_once_with('rst_link_dir/lnname.nc') + mock_isfile.assert_called_once_with('init_dir/fname.nc') + mock_slink.assert_not_called() + mock_stdout.assert_called_once_with( + '[INFO] No MODEL sub-PE restarts found\n') + +class TestSetupRestart(unittest.TestCase): + ''' + Test the top level of the setup restart functionality + ''' + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib._setup_multiple_restart') + @mock.patch('ocn_lib._setup_single_restart') + def test_setup_restart_multiple(self, mock_single, mock_multiple, + mock_nproc, mock_linkdir): + '''Setup the restart for multiple restart files''' + mock_nproc.return_value = 2 + ocn_lib.setup_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', + 'MODEL') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_multiple.assert_called_once_with( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', 2) + mock_single.assert_not_called() + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib._setup_multiple_restart') + @mock.patch('ocn_lib._setup_single_restart') + def test_setup_restart_single(self, mock_single, mock_multiple, + mock_nproc, mock_linkdir): + '''Setup the restart for single restart files''' + mock_nproc.return_value = 0 + ocn_lib.setup_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', + 'MODEL') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_single.assert_called_once_with( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_multiple.assert_not_called() + +class TestSetupNRUN(unittest.TestCase): + ''' + Test setting up of restart files for an NRUN + ''' + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib.sys.stdout.write') + def test_setup_nrun_unset_start_envar(self, mock_stdout, mock_linkdir): + '''Test a value for the START environment variable which evaluates to + false''' + self.assertEqual(ocn_lib.setup_nrun( + False, 'rst_link_dir', None, None, 'my_warning'), '.false.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_stdout.assert_called_once_with('[WARN] my_warning\n') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.sys.stderr.write') + def test_setup_nrun_no_restart( + self, mock_stderr, mock_isfile, mock_nproc, mock_linkdir): + '''Check we exit correctly if the restart file is not found''' + mock_nproc.return_value = 0 + mock_isfile.side_effect = [False, False] + with self.assertRaises(SystemExit) as context: + ocn_lib.setup_nrun('start', 'rst_link_dir', None, None, None) + self.assertEqual(context.exception.code, + error.MISSING_MODEL_FILE_ERROR) + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_stderr.assert_called_once_with('[FAIL] file start not found\n') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_setup_nrun_single_file( + self, mock_slink, mock_isfile, mock_nproc, mock_linkdir): + '''Check the linking of a single restart file for NRUN''' + mock_nproc.return_value = 0 + mock_isfile.return_value = True + self.assertEqual(ocn_lib.setup_nrun( + 'start', 'rst_link_dir', 'restart', 'init', None), '.true.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_nproc.assert_called_once_with('init') + mock_slink.assert_called_once_with('start', 'rst_link_dir/restart.nc') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.glob.glob') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.symlink') + def test_setup_nrun_multiple_file(self, mock_slink, mock_rmfile, mock_glob, + mock_isfile, mock_nproc, mock_linkdir): + '''Check the linking of multiple restart files for NRUN''' + mock_nproc.return_value = 4 + mock_isfile.side_effect = [False, True] + mock_glob.return_value = ['start_0000.nc', 'start_0001.nc'] + self.assertEqual(ocn_lib.setup_nrun( + 'start', 'rst_link_dir', 'restart', 'init', None), '.true.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_nproc.assert_called_once_with('init') + mock_isfile.assert_has_calls([mock.call('start'), + mock.call('start_0000.nc')]) + mock_glob.assert_called_once_with('start_????.nc') + mock_rmfile.assert_has_calls( + [mock.call('rst_link_dir/restart_0000.nc'), + mock.call('rst_link_dir/restart_0001.nc')]) + mock_slink.assert_has_calls( + [mock.call('start_0000.nc', 'rst_link_dir/restart_0000.nc'), + mock.call('start_0001.nc', 'rst_link_dir/restart_0001.nc')]) + +class TestCreateRestartLinkDir(unittest.TestCase): + ''' + Test the functionaility of creating restart directory + ''' + @mock.patch('ocn_lib.os.mkdir') + def test_create_linkdir_wd(self, mock_mkdir): + '''Test that nothing happens if the restart dir is the current + working dir '.' ''' + self.assertIsNone(ocn_lib._create_restart_link_dir('.')) + mock_mkdir.assert_not_called() + + @mock.patch('ocn_lib.os.mkdir') + def test_create_linkdir(self, mock_mkdir): + '''Test that we call os.mkdir when the directory doesnt exist''' + self.assertIsNone(ocn_lib._create_restart_link_dir('restart_dir')) + mock_mkdir.assert_called_once_with('restart_dir') + + @mock.patch('ocn_lib.os.mkdir') + @mock.patch('ocn_lib.sys.stdout.write') + def test_create_linkdir_existing_folder(self, mock_stdout, mock_mkdir): + '''Test that a FileExistsError is captured if dictionary can't be + created''' + mock_mkdir.side_effect = FileExistsError + self.assertIsNone(ocn_lib._create_restart_link_dir('restart_dir')) + mock_stdout.assert_called_once_with( + '[INFO] Restart link subdirectory restart_dir exists\n') From e2b8bbf82647fc6a4303bf13659411ff3ae29a40 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:24:54 +0100 Subject: [PATCH 35/47] Add update nemo nl tests Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- .../unittests/test_update_nemo_nl.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 Coupled_Drivers/unittests/test_update_nemo_nl.py diff --git a/Coupled_Drivers/unittests/test_update_nemo_nl.py b/Coupled_Drivers/unittests/test_update_nemo_nl.py new file mode 100644 index 00000000..562f7d8f --- /dev/null +++ b/Coupled_Drivers/unittests/test_update_nemo_nl.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' +import os +import unittest +import unittest.mock as mock + +import update_nemo_nl + +class TestUpdateSI3NL(unittest.TestCase): + ''' + Test the update SI3 NL file + ''' + def setUp(self): + '''Set up the unit test''' + # The internal variables dictionary + self.variables = {'cn_icerst_indir': "'ocean_link_directory'"} + self.nemo_envar = {'SI3_NL': 'si3_namelist', + 'RST_LINK_DIR': 'ocean_link_directory'} + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_si3(self, mock_move, mock_update): + '''Test the correct behaviour for updating SI3 namelist''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_si3_nl(self.nemo_envar) + mock_update.assert_called_once_with('si3_namelist', self.variables) + mock_move.assert_called_once_with('my_nl_file.tmp', 'si3_namelist') + +class TestUpdateTopNL(unittest.TestCase): + ''' + Test the update Top NL file + ''' + def setUp(self): + '''Set up the variables dictionary''' + # The internal variables dictionary + self.variables = {'ln_rsttr': 'my_ln_restart', + 'nn_rsttr': 'my_restart_ctl', + 'ln_trcdta': 'my_ln_trcdta', + 'cn_trcrst_indir': "'ocean_link_directory'"} + self.nemo_envar = {'TOP_NL': 'top_namelist', + 'RST_LINK_DIR': 'ocean_link_directory'} + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_top(self, mock_move, mock_update): + '''Test the correct behaviour for updating top namelist''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_top_nl(self.nemo_envar, 'my_ln_restart', + 'my_restart_ctl', 'my_ln_trcdta') + mock_update.assert_called_once_with('top_namelist', self.variables) + mock_move.assert_called_once_with('my_nl_file.tmp', 'top_namelist') + + +class TestUpdateNemoNL(unittest.TestCase): + ''' + Test the update nemo nl file, with NEMO3.6 and NEMO4 behaviour + ''' + def setUp(self): + '''Set up the variables dictionary''' + # The internal variable dictionaries + self.variables_4 = {'cn_exp': "'my_runido'", + 'ln_rstart': 'my_restart', + 'nn_rstctl': 'my_ctl', + 'nn_it000': 'my_next_step', + 'nn_itend': 'my_final_step', + 'nn_date0': 'my_ndate0', + 'nn_leapy': 'my_leapy', + 'jpni': 'my_iproc', + 'jpnj': 'my_jproc', + 'nn_cpl_river': 'my_cpl_river', + 'cn_ocerst_indir': "'ocean_link_directory'"} + self.variables_36 = {**self.variables_4, **{'jpnij': 'my_jpnij'}} + + # Common envar to be passed in + self.common_envar = {'RUNID': 'my_runid', + 'CPL_RIVER_COUNT': 'my_cpl_river'} + + # Nemo environment variables to be passed in + self.nemo_envar_4 = {'NEMO_VERSION': 400, + 'NEMO_IPROC': 'my_iproc', + 'NEMO_JPROC': 'my_jproc', + 'NEMO_NL': 'my_nl_file', + 'RST_LINK_DIR': 'ocean_link_directory'} + # Sort out for nemo 3.6 + self.nemo_envar_36 = {**self.nemo_envar_4, **{'NEMO_NPROC': 'my_jpnij'}} + self.nemo_envar_36['NEMO_VERSION'] = 360 + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_nl_nemo_4(self, mock_move, mock_update): + '''Test the correct behaviour for NEMO4''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_nl(self.common_envar, self.nemo_envar_4, + 'my_restart', 'my_ctl', 'my_next_step', + 'my_final_step', 'my_ndate0', 'my_leapy') + mock_update.assert_called_with('my_nl_file', self.variables_4) + mock_move.assert_called_with('my_nl_file.tmp', 'my_nl_file') + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_nl_nemo_36(self, mock_move, mock_update): + '''Test the correct behaviour for NEMO3.6''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_nl(self.common_envar, self.nemo_envar_36, + 'my_restart', 'my_ctl', 'my_next_step', + 'my_final_step', 'my_ndate0', 'my_leapy') + mock_update.assert_called_with('my_nl_file', self.variables_36) + mock_move.assert_called_with('my_nl_file.tmp', 'my_nl_file') + + +class TestDoUpdateNLFiles(unittest.TestCase): + ''' + Test the updating of namelist files + ''' + def setUp(self): + '''Create a dummy namelist file to update''' + self.nl_name = 'dummy_nl_cfg' + self.nl_swap_name = 'dummy_nl_cfg.tmp' + # Create our input file + file_contents = '''&namrun +swap1=original_value1, +ln_mskland=.true., +/ +&namzgr +swap2='original_value2', +swap3=original_value3, +swap4=original_value4, +ln_zps=.true., +/ +''' + with open(self.nl_name, 'w') as fh: + fh.write(file_contents) + + # Expected contents after swap + self.expected_out = '''&namrun +swap1=new_value1, +ln_mskland=.true., +/ +&namzgr +swap2=.true., +swap3=.false., +swap4=4.5, +ln_zps=.true., +/ +''' + + def tearDown(self): + '''Remove any files created in the unit test''' + for filename in [self.nl_name, self.nl_swap_name]: + try: + os.remove(filename) + except FileNotFoundError: + pass + + def test_update_nl_files(self): + '''Test the updating function''' + variables = {'swap1': 'new_value1', + 'swap2': '.true.', + 'swap3': '.false.', + 'swap4': 4.5} + rvalue = update_nemo_nl.do_update_nl_files(self.nl_name, variables) + self.assertEqual(rvalue, self.nl_swap_name) + with open(self.nl_swap_name, 'r') as fh: + for line, expected_line in zip(fh.readlines(), + self.expected_out.split('\n')): + self.assertEqual(line.rstrip(), expected_line) + From 4738b9278e559d023aee3557bf7020a62db05c75 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 13:33:20 +0100 Subject: [PATCH 36/47] Add python to update nemo nl Co-authored-by: harry-shepherd <17930806+harry-shepherd@users.noreply.github.com> --- Coupled_Drivers/update_nemo_nl.py | 91 +++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 Coupled_Drivers/update_nemo_nl.py diff --git a/Coupled_Drivers/update_nemo_nl.py b/Coupled_Drivers/update_nemo_nl.py new file mode 100644 index 00000000..5229d9ee --- /dev/null +++ b/Coupled_Drivers/update_nemo_nl.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + update_nemo_nl.py + +DESCRIPTION + Update the NEMO namelists. Valid for NEMO 3.6 and NEMO 4.0. The interface + function is update_nl +''' + +import shutil +import common + +def do_update_nl_files(namelist_filename, variables): + ''' + Do the replacement of the variables in the namelist file and + return the filename of the new file + ''' + updated_nl_file = '%s.tmp' % namelist_filename + _ = common.remove_file(updated_nl_file) + with common.open_text_file(updated_nl_file, 'w') as new_fh: + with common.open_text_file(namelist_filename, 'r') as old_fh: + for line in old_fh.readlines(): + if '=' in line: + varname = line.split('=')[0] + if varname in variables.keys(): + newline = '%s=%s,\n' % (varname, variables[varname]) + new_fh.write(newline) + else: + new_fh.write(line) + else: + new_fh.write(line) + return updated_nl_file + +def update_si3_nl(nemo_envar): + ''' + Update the SI3 namelist + ''' + variables = {'cn_icerst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + updated_nl_filename = do_update_nl_files(nemo_envar['SI3_NL'], variables) + shutil.move(updated_nl_filename, nemo_envar['SI3_NL']) + +def update_top_nl(nemo_envar, ln_restart, restart_ctl, ln_trcdta): + ''' + Update the Top namelist + ''' + variables = {'ln_rsttr': ln_restart, + 'nn_rsttr': restart_ctl, + 'ln_trcdta': ln_trcdta, + 'cn_trcrst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + updated_nl_filename = do_update_nl_files(nemo_envar['TOP_NL'], variables) + shutil.move(updated_nl_filename, nemo_envar['TOP_NL']) + + +def update_nl(common_envar, nemo_envar, ln_restart, restart_ctl, nemo_next_step, + nemo_final_step, nemo_ndate0, nleapy): + ''' + Update the NEMO namelist + ''' + # variables to replace for NEMO3.6 and NEMO4.0 + # The run ID needs to appear in quotes in the namelist file and takes the + # letter 'o' as a suffix (cn_exp) + variables = {'cn_exp': "'%so'" % common_envar['RUNID'], + 'ln_rstart': ln_restart, + 'nn_rstctl': restart_ctl, + 'nn_it000': nemo_next_step, + 'nn_itend': nemo_final_step, + 'nn_date0': nemo_ndate0, + 'nn_leapy': nleapy, + 'jpni': nemo_envar['NEMO_IPROC'], + 'jpnj': nemo_envar['NEMO_JPROC'], + 'nn_cpl_river': common_envar['CPL_RIVER_COUNT'], + 'cn_ocerst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + # add nemo3.6 only variable + if int(nemo_envar['NEMO_VERSION']) < 400: + variables['jpnij'] = nemo_envar['NEMO_NPROC'] + + updated_nl_filename = do_update_nl_files(nemo_envar['NEMO_NL'], + variables) + shutil.move(updated_nl_filename, nemo_envar['NEMO_NL']) From cbf2b23f3fc198a14e45334276e9402d16673026 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 14:00:53 +0100 Subject: [PATCH 37/47] Add removal of trailing slashes to common --- Coupled_Drivers/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index cb553b2e..8ebae078 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -186,6 +186,9 @@ def remove_file(filename): else: return False +def remove_trailing_slash(path_str): + return path_str.rstrip(' /') + def setup_runtime(common_env): ''' Set up model run length in seconds based on the model suite From bde824beada8ea589ca237992c34aebd5e3f2f6f Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 14:51:18 +0100 Subject: [PATCH 38/47] Update ocn_cont_def environment variables --- Coupled_Drivers/dr_env_lib/ocn_cont_def.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py index bd66e8ff..d8038b55 100644 --- a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py +++ b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py @@ -18,11 +18,11 @@ Definition of the environment variables required for the ocean controllers ''' - +SI3_ENVIRONMENT_VARS_COMMON = {'SI3_NL': {'default_val': 'namelist_ice_cfg'}} SI3_ENVIRONMENT_VARS_INITIAL = { 'SI3_START': {'default_val': ''}, 'SI3_NL': {'default_val': 'namelist_ice_cfg'}} - +SI3_ENVIRONMENT_VARS_FINAL = SI3_ENVIRONMENT_VARS_COMMON TOP_ENVIRONMENT_VARS_COMMON = { 'TOP_NL': {'default_val': 'namelist_top_cfg'}} TOP_ENVIRONMENT_VARS_INITIAL = { @@ -32,3 +32,4 @@ TOP_ENVIRONMENT_VARS_INITIAL = {**TOP_ENVIRONMENT_VARS_COMMON, **TOP_ENVIRONMENT_VARS_INITIAL} +TOP_ENVIRONMENT_VARS_FINAL = TOP_ENVIRONMENT_VARS_COMMON From b34b63cb07e3ec6cea5c7a26dc4fe6022d7bc967 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 15:08:27 +0100 Subject: [PATCH 39/47] Add missing functions which fail nemo unit tests back --- Coupled_Drivers/nemo_driver.py | 86 ++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index d5603ff2..902dea2c 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -36,6 +36,92 @@ # Define errors for the NEMO driver only SERIAL_MODE_ERROR = 99 +def _verify_fix_rst(restartdate, nemo_rst, model_basis_time, time_step, + num_steps, calendar): + ''' + Verify that the restart file for nemo corresponds to the model + time reached within a given model run. If they don't match, then + make sure that nemo restarts from the correct restart date + + :arg: string restartdate : NEMO dump file date + :arg: string nemo_rst : Path to NEMO restart files + :arg: string model_basis_time : Model basis time + :arg: int time_step : Ocean time-step (in seconds) + :arg: int num_steps : Num. of time-steps covered + :arg: string calendar : Calendar used in model + + ''' + # Calculate the model restart time based on the start date of the + # last calculated model step, the time-step and the number of + # steps. Then convert the date format. + model_basis_datetime = datetime.datetime.strptime( + model_basis_time, "%Y%m%d") + + model_restart_date = _calc_current_model_date( + model_basis_datetime, time_step, num_steps, calendar) + + model_restart_date = model_restart_date.strftime('%Y%m%d') + + if restartdate == model_restart_date: + sys.stdout.write('[INFO] Validated NEMO restart date\n') + else: + # Write the message to both standard out and standard error + msg = '[WARN] The NEMO restart data does not match the ' \ + ' current model time\n.' \ + ' Current model date is %s\n' \ + ' NEMO restart time is %s\n' \ + '[WARN] Automatically removing NEMO dumps ahead of ' \ + 'the current model date, and pick up the dump at ' \ + 'this time\n' % (model_restart_date, restartdate) + sys.stdout.write(msg) + sys.stderr.write(msg) + #Remove all nemo restart files that are later than the correct + #cycle times + #Make our generic restart regular expression, to cover normal NEMO + #restart, and potential iceberg, SI3 or passive tracer restart files, + #for both the rebuilt and non rebuilt cases + generic_rst_regex = r'(icebergs)?.*restart(_trc)?(_ice)?(_icb)?(_\d+)?\.nc' + all_restart_files = [f for f in os.listdir(nemo_rst) if + re.findall(generic_rst_regex, f)] + for restart_file in all_restart_files: + fname_date = re.findall(r'\d{8}', restart_file)[0] + if fname_date > model_restart_date: + common.remove_file(os.path.join(nemo_rst, restart_file)) + restartdate = model_restart_date + + return restartdate + +def _calc_current_model_date(model_basis_time, time_step, num_steps, + calendar): + ''' + Calculate the current model date using the basis time, + and the number of time-steps covered in a given model run. + + :arg: datetime model_basis_time : Model basis time + :arg: int time_step : Ocean model timestep (in seconds) + :arg: int num_steps : Num. timesteps covered in this + model run + :arg: string calendar : Calendar used in the model run + + ''' + ref_date_format = 'seconds since %Y-%m-%d %H:%M:%S' + + # modify the calendar names for compatability with cf_units module + if calendar == "360day": + calendar = "360_day" + if calendar == "365day": + calendar = "365_day" + + # Provide a reference time for the timestep incrementation. + ref_time = model_basis_time.strftime(ref_date_format) + + model_progress_secs = cf_units.date2num( + model_basis_time, ref_time, calendar=calendar) + (time_step * num_steps) + + current_model_date = cf_units.num2date( + model_progress_secs, ref_time, calendar=calendar) + + return current_model_date def _nemo_driver_assertions(nemo_envar): ''' From e53114659de82ff67535e8fea47a5ab67b70a1e9 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 27 May 2026 15:11:05 +0100 Subject: [PATCH 40/47] Update nemo driver imports --- Coupled_Drivers/nemo_driver.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index 902dea2c..27cc5aeb 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -18,15 +18,41 @@ Driver for the NEMO 3.6 model, called from link_drivers. Note that this does not cater for any earlier versions of NEMO ''' -import collections -import importlib -import os import re -import shutil +import os +import time +import datetime import sys +import glob +import shutil +import collections +import importlib +import inc_days import common import error + +try: + import cf_units +except ImportError: + IMPORT_ERROR_MSG = ('Unable to import cf_units. Ensure the scitools module' + 'has been loaded first.') + sys.exit(IMPORT_ERROR_MSG) + +import dr_env_lib.nemo_def +import dr_env_lib.env_lib +try: + import f90nml +except ImportError: + pass + +# Here, "top" refers to the NEMO TOP passive tracer system. It does not +# imply anything to do with being in overall control or at the head of +# any form of control heirarchy. +import top_controller + +import si3_controller + import nemo_lib import nemo_restart_lib import nemo_runtime_namcouple From 38b3b66a83013b1ea72fa093080de2dab24b72e3 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Thu, 28 May 2026 13:23:03 +0100 Subject: [PATCH 41/47] Add trailing whitespace removal --- Coupled_Drivers/common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index 8ebae078..72f143ec 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -187,7 +187,10 @@ def remove_file(filename): return False def remove_trailing_slash(path_str): - return path_str.rstrip(' /') + """ + Remove any trailing slashes and whitespace from the path string. + """ + return path_str.rstrip(' \t\n\r/') def setup_runtime(common_env): ''' From 44d1043842c5335698d3d6ef50696358054872f5 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Mon, 1 Jun 2026 11:35:51 +0100 Subject: [PATCH 42/47] Fix nemo driver unit tests --- Coupled_Drivers/unittests/test.py | 1 - Coupled_Drivers/unittests/test_nemo_driver.py | 4 +-- .../unittests/test_nemo_driver_submod.py | 26 +++++++++++-------- rose-stem/site/meto_azspice.cylc | 4 +-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Coupled_Drivers/unittests/test.py b/Coupled_Drivers/unittests/test.py index da44b34a..6cc8a164 100755 --- a/Coupled_Drivers/unittests/test.py +++ b/Coupled_Drivers/unittests/test.py @@ -45,7 +45,6 @@ def main(): 'xios': 'test_xios_driver.py', 'dependency_checker': 'test_driver_dependencies.py', 'cpmip': 'test_cpmip*py', - 'nemo_driver': 'test_nemo_driver.py', 'um_driver': 'test_um_driver.py', 'cice_driver': 'test_cice_driver.py', 'rivers': 'test_rivers_driver.py', diff --git a/Coupled_Drivers/unittests/test_nemo_driver.py b/Coupled_Drivers/unittests/test_nemo_driver.py index 88cf1308..c49382a2 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver.py +++ b/Coupled_Drivers/unittests/test_nemo_driver.py @@ -146,7 +146,7 @@ def test_verify_fix_rst_success(self): restart_date = '19931116' expected_msg = '[INFO] Validated NEMO restart date\n' - with mock.patch('sys.stdout', new=io.StringIO()) as output: + with mock.patch('nemo_driver.sys.stdout', new=io.StringIO()) as output: corrected_restart_date = nemo_driver._verify_fix_rst( restart_date, self.nemo_rst, self.model_step_start, self.nemo_step_int, self.nemo_last_step, self.calendar) @@ -174,7 +174,7 @@ def test_verify_fix_rst_fail(self): 'cplhco_19931116_restart_icb_0050.nc' ] - with mock.patch('sys.stdout', new=io.StringIO()) as output: + with mock.patch('nemo_driver.sys.stdout', new=io.StringIO()) as output: corrected_restart_date = nemo_driver._verify_fix_rst( restart_date, self.nemo_rst, self.model_step_start, self.nemo_step_int, self.nemo_last_step, self.calendar) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_submod.py b/Coupled_Drivers/unittests/test_nemo_driver_submod.py index a4115d36..b82f71e3 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver_submod.py +++ b/Coupled_Drivers/unittests/test_nemo_driver_submod.py @@ -12,7 +12,6 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** ''' - import unittest import unittest.mock as mock @@ -39,21 +38,25 @@ def clear(self): self.top_controller = MockController('TOP') self.si3_controller = MockController('SI3') - @mock.patch('nemo_driver.sys.stdout.write') @mock.patch('nemo_driver.importlib.import_module') - def test_no_controllers(self, mock_import, mock_stdout): - '''Test the code runs correctly if no controllers are used''' + @mock.patch('nemo_driver.sys.stdout') + def test_no_controllers(self, mock_stdout, mock_import): nemo_envar = {'L_OCN_PASS_TRC': 'False'} common_envar = {'models': 'um nemo'} - nemo_driver._run_submodel_controllers(nemo_envar, common_envar, - None, None, None) - mock_stdout.assert_called_once_with( - '[INFO] nemo_driver: Passive tracer code not active.\n') + + nemo_driver._run_submodel_controllers( + nemo_envar, common_envar, None, None, None + ) + + mock_stdout.write.assert_called_once_with( + '[INFO] nemo_driver: Passive tracer code not active.\n' + ) mock_import.assert_not_called() - @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.importlib.import_module') - def test_top_only(self, mock_import, mock_stdout): + @mock.patch('nemo_driver.sys.stdout') + def test_top_only(self, mock_stdout, mock_import): '''Test that a top only call works, with upper case T''' nemo_envar = {'L_OCN_PASS_TRC': 'True', 'NEMO_NPROC': '10'} @@ -65,10 +68,11 @@ def test_top_only(self, mock_import, mock_stdout): 'verify_rst', 'nemo_dump_time', 'mode') self.top_controller.expected_arguments = expected_args mock_import.return_value = self.top_controller + nemo_driver._run_submodel_controllers(nemo_envar, common_envar, 'restart_ctl', 'nemo_dump_time', 'mode') - mock_stdout.assert_has_calls( + mock_stdout.write.assert_has_calls( [mock.call('[INFO] nemo_driver: Passive tracer code is active.\n'), mock.call('[INFO] Calling top_controller in mode mode\n')]) mock_import.assert_called_once_with('top_controller') diff --git a/rose-stem/site/meto_azspice.cylc b/rose-stem/site/meto_azspice.cylc index 1e0481e3..579042a7 100644 --- a/rose-stem/site/meto_azspice.cylc +++ b/rose-stem/site/meto_azspice.cylc @@ -18,7 +18,7 @@ [[[environment]]] PYTHON_ENVIRONMENT = """ echo *** Linux Python Environment Begin *** -module load scitools scitools/default-current +module load scitools/default-current module load um_tools echo $(module list) echo *** Linux Python Environment End *** @@ -40,7 +40,7 @@ echo *** Linux Python Environment End *** {% if USE_LINUX_SERVER %} [[[directives]]] --mem=10G - {% endif %} + {% endif %} {% endif %} [[DRIVERS_TESTS_RESOURCE]] From bcb9431085fbf48cdb83c1653499e20a09f87824 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 2 Jun 2026 10:25:06 +0100 Subject: [PATCH 43/47] Add unit test skeletons --- Coupled_Drivers/unittests/test_nemo_driver_misc.py | 8 ++++++++ Coupled_Drivers/unittests/test_ocn_lib.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py index ad46951d..2b134e81 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver_misc.py +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -591,3 +591,11 @@ def test_verify_restart(self, mock_verify_fix): mock_verify_fix.assert_called_once_with( 'nemo_dump_time', 'cycle_point', 'nemo_rst') +class TestSetLauncherCommand(unittest.TestCase): + + def test_set_launcher_command_duff(self): + pass + + def test_set_launcher_command_non_duff(self): + pass + diff --git a/Coupled_Drivers/unittests/test_ocn_lib.py b/Coupled_Drivers/unittests/test_ocn_lib.py index d5b6f560..7315aa42 100644 --- a/Coupled_Drivers/unittests/test_ocn_lib.py +++ b/Coupled_Drivers/unittests/test_ocn_lib.py @@ -101,6 +101,11 @@ def test_read_variables_fail(self, mock_stderr): ' from file %s\n' % (self.nl_name)) del read_ocn_namelist_object + def test_read_known_file(self): + pass + + def test_read_unknown_file(self): + pass From 899a9dfa19a180447e517a1c93274e9f8a93acee Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:37:23 +0100 Subject: [PATCH 44/47] Setup bare bones of setup launcher command tests --- Coupled_Drivers/unittests/test_nemo_driver_misc.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py index 2b134e81..a2bf02e4 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver_misc.py +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -593,9 +593,15 @@ def test_verify_restart(self, mock_verify_fix): class TestSetLauncherCommand(unittest.TestCase): - def test_set_launcher_command_duff(self): - pass + @mock.patch('nemo_driver._setup_executable') + def test_set_launcher_command_duff(self, mock_setup_executable): + mock_setup_executable.return_value = None + common_env = {'ROSE_LAUNCHER': 'launcher'} + exe_envar = nemo_driver.setup_executable(common_env) - def test_set_launcher_command_non_duff(self): - pass + @mock.patch('nemo_driver._setup_executable') + def test_set_launcher_command_non_duff(self, mock_setup_executable): + mock_setup_executable.return_value = None + common_env = {'ROSE_LAUNCHER': 'launcher'} + exe_envar = nemo_driver.setup_executable(common_env) From 786628c9801a97c5671a5f95b6a9b486f913456d Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:02:57 +0100 Subject: [PATCH 45/47] Add set launcher command unit tests --- .../unittests/test_nemo_driver_misc.py | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py index a2bf02e4..3b9b10b0 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver_misc.py +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -592,16 +592,59 @@ def test_verify_restart(self, mock_verify_fix): 'nemo_dump_time', 'cycle_point', 'nemo_rst') class TestSetLauncherCommand(unittest.TestCase): - - @mock.patch('nemo_driver._setup_executable') - def test_set_launcher_command_duff(self, mock_setup_executable): - mock_setup_executable.return_value = None - common_env = {'ROSE_LAUNCHER': 'launcher'} - exe_envar = nemo_driver.setup_executable(common_env) - - @mock.patch('nemo_driver._setup_executable') - def test_set_launcher_command_non_duff(self, mock_setup_executable): - mock_setup_executable.return_value = None - common_env = {'ROSE_LAUNCHER': 'launcher'} - exe_envar = nemo_driver.setup_executable(common_env) + ''' + Test the setting up of the launcher command for NEMO executable + ''' + @mock.patch('nemo_driver.common.set_aprun_options') + def test_set_launcher_command_aprun_unset(self, mock_set_aprun): + '''Test with aprun launcher and ROSE_LAUNCHER_PREOPTS_NEMO unset''' + mock_set_aprun.return_value = 'aprun -n 240 -N 24 -S 12 -d 1' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'unset', + 'NEMO_NPROC': '240', + 'OCEAN_NODES': '10', + 'OMPTHR_OCN': '1', + 'OHYPERTHREADS': 'false', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'aprun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command + self.assertEqual(result, 'aprun -n 240 -N 24 -S 12 -d 1 ./nemo.exe') + # Check that nemo_envar was updated and quoted + self.assertEqual( + nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], + "'aprun -n 240 -N 24 -S 12 -d 1'") + # Check that set_aprun_options was called correctly + mock_set_aprun.assert_called_once_with('240', '10', '1', 'false', False) + + def test_set_launcher_command_non_aprun_unset(self): + '''Test with non-aprun launcher and ROSE_LAUNCHER_PREOPTS_NEMO unset''' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'unset', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'srun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command (empty preopts) + self.assertEqual(result, ' ./nemo.exe') + # Check that nemo_envar was updated and quoted + self.assertEqual(nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], "''") + + def test_set_launcher_command_preopts_already_set(self): + '''Test when ROSE_LAUNCHER_PREOPTS_NEMO is already configured''' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'mpirun -np 100', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'aprun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command uses existing preopts + self.assertEqual(result, 'mpirun -np 100 ./nemo.exe') + # Check that nemo_envar was quoted + self.assertEqual( + nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], "'mpirun -np 100'") From 490f1a7be3ff9844b53b751a7444cc5d6fe92949 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:09:08 +0100 Subject: [PATCH 46/47] Restyle white space --- Coupled_Drivers/unittests/test_nemo_driver_misc.py | 1 - Coupled_Drivers/unittests/test_ocn_lib.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py index 3b9b10b0..f4417523 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver_misc.py +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -647,4 +647,3 @@ def test_set_launcher_command_preopts_already_set(self): # Check that nemo_envar was quoted self.assertEqual( nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], "'mpirun -np 100'") - diff --git a/Coupled_Drivers/unittests/test_ocn_lib.py b/Coupled_Drivers/unittests/test_ocn_lib.py index 7315aa42..5d0c01e6 100644 --- a/Coupled_Drivers/unittests/test_ocn_lib.py +++ b/Coupled_Drivers/unittests/test_ocn_lib.py @@ -101,14 +101,6 @@ def test_read_variables_fail(self, mock_stderr): ' from file %s\n' % (self.nl_name)) del read_ocn_namelist_object - def test_read_known_file(self): - pass - - def test_read_unknown_file(self): - pass - - - class TestGetNemoNprocStr(unittest.TestCase): ''' Unit tests for determining the number of zeros in the unrebuilt NEMO From 800223ca4657db7fe590e48e409073488d1a00c8 Mon Sep 17 00:00:00 2001 From: Pierre Siddall <43399998+Pierre-siddall@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:14:50 +0100 Subject: [PATCH 47/47] remove unused imports --- Coupled_Drivers/nemo_driver.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index 27cc5aeb..f4feb87b 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -23,12 +23,9 @@ import time import datetime import sys -import glob import shutil import collections import importlib - -import inc_days import common import error @@ -38,21 +35,7 @@ IMPORT_ERROR_MSG = ('Unable to import cf_units. Ensure the scitools module' 'has been loaded first.') sys.exit(IMPORT_ERROR_MSG) - -import dr_env_lib.nemo_def -import dr_env_lib.env_lib -try: - import f90nml -except ImportError: - pass - -# Here, "top" refers to the NEMO TOP passive tracer system. It does not -# imply anything to do with being in overall control or at the head of -# any form of control heirarchy. -import top_controller - -import si3_controller - + import nemo_lib import nemo_restart_lib import nemo_runtime_namcouple