diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index 538f17ac..72f143ec 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2022-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,8 +18,6 @@ Common functions and classes required by multiple model drivers ''' - - import datetime import glob import shutil @@ -32,7 +30,6 @@ import error import inc_days - class ModNamelist(object): ''' Modify a fortran namelist. This will not add any new variables, only @@ -49,7 +46,7 @@ def __init__(self, filename): def var_val(self, variable, value): ''' Create a container of variable name, value pairs to be updated. Note - that if a variable doesn't exist in the namelist file, then it + that if a variable doesn't exisit in the namelist file, then it will be ignored ''' if isinstance(value, str): @@ -139,6 +136,7 @@ def get_filepaths(directory): return file_paths + def open_text_file(name, mode): ''' Provide a common function to open a file and provide a suitiable error @@ -152,7 +150,7 @@ def open_text_file(name, mode): 'a+':'updating (appending)'} if mode not in list(modes.keys()): options = '' - for k in modes: + for k in modes.keys(): options += ' %s: %s\n' % (k, modes[k]) sys.stderr.write('[FAIL] Attempting to open file %s, do not recognise' ' mode %s. Please use one of the following modes:\n%s' @@ -188,6 +186,12 @@ def remove_file(filename): else: return False +def remove_trailing_slash(path_str): + """ + Remove any trailing slashes and whitespace from the path string. + """ + return path_str.rstrip(' \t\n\r/') + def setup_runtime(common_env): ''' Set up model run length in seconds based on the model suite diff --git a/Coupled_Drivers/dr_env_lib/nemo_def.py b/Coupled_Drivers/dr_env_lib/nemo_def.py index 98e78b2a..0f1cf04f 100644 --- a/Coupled_Drivers/dr_env_lib/nemo_def.py +++ b/Coupled_Drivers/dr_env_lib/nemo_def.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -22,18 +22,20 @@ # Variables required for both initalise and finalise NEMO_ENVIRONMENT_VARS_COMMON = { 'NEMO_NL': {'default_val': 'namelist_cfg'}, - 'L_OCN_PASS_TRC': {'default_val': 'false'} + 'L_OCN_PASS_TRC': {'default_val': 'false'}, + 'NEMO_NPROC': {'desc': 'Number of NEMO processors'}, + 'RST_LINK_DIR': {'default_val': '.', + 'desc': 'Subdirectory of $CYCL_TASK_WORK_DIR containing' \ + ' ocean restart links'} } NEMO_ENVIRONMENT_VARS_INITIAL = { 'OCEAN_EXEC': {'desc': ('Ocean executable (OCEAN_EXEC=)')}, - 'NEMO_NPROC': {'desc': 'Number of NEMO processors'}, 'NEMO_IPROC': {'desc': 'Number of NEMO processors in the i direction'}, 'NEMO_JPROC': {'desc': 'Number of NEMO processors in the j direction'}, 'NEMO_VERSION': {}, 'OCEAN_LINK': {'default_val': 'ocean.exe'}, - 'OCN_RES': {'default_val': ''}, 'NEMO_NL': {'default_val': 'namelist_cfg'}, 'NEMO_START': {'default_val': ''}, 'NEMO_ICEBERGS_START': {'default_val': ''}, diff --git a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py index 6acf0563..d8038b55 100644 --- a/Coupled_Drivers/dr_env_lib/ocn_cont_def.py +++ b/Coupled_Drivers/dr_env_lib/ocn_cont_def.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,11 +18,11 @@ Definition of the environment variables required for the ocean controllers ''' - +SI3_ENVIRONMENT_VARS_COMMON = {'SI3_NL': {'default_val': 'namelist_ice_cfg'}} SI3_ENVIRONMENT_VARS_INITIAL = { 'SI3_START': {'default_val': ''}, 'SI3_NL': {'default_val': 'namelist_ice_cfg'}} - +SI3_ENVIRONMENT_VARS_FINAL = SI3_ENVIRONMENT_VARS_COMMON TOP_ENVIRONMENT_VARS_COMMON = { 'TOP_NL': {'default_val': 'namelist_top_cfg'}} TOP_ENVIRONMENT_VARS_INITIAL = { diff --git a/Coupled_Drivers/error.py b/Coupled_Drivers/error.py index 5e4d69a3..a9d026c3 100644 --- a/Coupled_Drivers/error.py +++ b/Coupled_Drivers/error.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -53,6 +53,9 @@ # Invalid argument to driver script INVALID_DRIVER_ARG_ERROR = 202 +# Missing file contents +MISSING_FILE_CONTENTS_ERROR = 203 + # Missing file required by a controller MISSING_CONTROLLER_FILE_ERROR = 251 @@ -139,6 +142,9 @@ # Unable to create remapping file for mapping type MISSING_RMP_MAPPING = 816 +# Size of 1D grid is unknown +UNKNOWN_1D_GRID_SIZE = 817 + # Ocean resolution for Snr<->Jnr coupling is not specified NO_OCN_RESOL = 818 diff --git a/Coupled_Drivers/fcm_make/driver.cfg b/Coupled_Drivers/fcm_make/driver.cfg index 2f63363b..d7f50d40 100644 --- a/Coupled_Drivers/fcm_make/driver.cfg +++ b/Coupled_Drivers/fcm_make/driver.cfg @@ -14,8 +14,8 @@ extract.ns = drivers extract.location{primary}[drivers] = fcm:moci.xm extract.location[drivers] = $driver_base${driver_rev} extract.path-excl[drivers] = / -extract.path-incl[drivers] = Coupled_Drivers Utilities/NGMS_utils/ngms_namcouple_gen Utilities/NGMS_utils/ngms_suite_lib +extract.path-incl[drivers] = Coupled_Drivers Utilities/NGMS_utils/ngms_namcouple_gen Utilities/NGMS_utils/ngms_suite_lib -$install_common= +$install_common = build.target = $install_common diff --git a/Coupled_Drivers/link_drivers b/Coupled_Drivers/link_drivers index e5cb3439..0046f219 100755 --- a/Coupled_Drivers/link_drivers +++ b/Coupled_Drivers/link_drivers @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -19,19 +19,24 @@ DESCRIPTION to run a given configuration ''' - +#The from __future__ imports ensure compatibility between python2.7 and 3.x +from __future__ import absolute_import import importlib import sys import os import argparse import error import common +import write_namcouple import dr_env_lib.common_def import dr_env_lib.env_lib #A tuple for for the earlist version of python required to run the drivers REQUIRED_VERSION = (3, 6) +# A submodel is a model which runs within another model, and doesnt require its # own driver or controller, for example si3 is a submodel of NEMO +SUBMODELS = ['si3', 'top'] + def _check_drivers(models): ''' Takes in our space separated list of models, and ensure that a driver is @@ -138,8 +143,6 @@ def _export_cmd_str(launchcmds, common_env): - - if __name__ == '__main__': # Owing to the limitations of the MPI launchers and passing information # between the shell and the python drivers, prepare two command files, one diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index e8167725..f4feb87b 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -23,9 +23,9 @@ import time import datetime import sys -import glob import shutil -import inc_days +import collections +import importlib import common import error @@ -35,144 +35,16 @@ IMPORT_ERROR_MSG = ('Unable to import cf_units. Ensure the scitools module' 'has been loaded first.') sys.exit(IMPORT_ERROR_MSG) - -import dr_env_lib.nemo_def -import dr_env_lib.env_lib -try: - import f90nml -except ImportError: - pass - -# Here, "top" refers to the NEMO TOP passive tracer system. It does not -# imply anything to do with being in overall control or at the head of -# any form of control heirarchy. -import top_controller - -import si3_controller + +import nemo_lib +import nemo_restart_lib +import nemo_runtime_namcouple +import ocn_lib +import update_nemo_nl # Define errors for the NEMO driver only SERIAL_MODE_ERROR = 99 -# Ocean resolutions -OCEAN_RESOLS = {'orca2': [182, 149], - 'orca1': [362, 332], - 'orca025': [1442, 1021], - 'orca12': [4322, 3059], - 'orca36': [12960, 10850]} - -def _check_nemonl(envar_container): - ''' - As the environment variable NEMO_NL is required by both the setup - and finalise functions, this will be encapsulated here - ''' - # Information will be retrieved from this file during the running of the - # driver, so check it exists - if not os.path.isfile(envar_container['NEMO_NL']): - sys.stderr.write('[FAIL] Can not find the nemo namelist file %s\n' % - envar_container['NEMO_NL']) - sys.exit(error.MISSING_DRIVER_FILE_ERROR) - else: - return 0 - -def _get_nemorst(nemo_nl_file): - ''' - Retrieve the nemo restart directory from the nemo namelist file - ''' - ocerst_rcode, ocerst_val = common.exec_subproc([ \ - 'grep', 'cn_ocerst_outdir', nemo_nl_file]) - if ocerst_rcode == 0: - nemo_rst = re.findall(r'[\"\'](.*?)[\"\']', ocerst_val)[0] - if nemo_rst[-1] == '/': - nemo_rst = nemo_rst[:-1] - return nemo_rst - return None - -def _get_ln_icebergs(nemo_nl_file): - ''' - Interrogate the nemo namelist to see if we are running with icebergs, - Returns boolean, True if icebergs are used, False if not - ''' - icb_rcode, icb_val = common.exec_subproc([ \ - 'grep', 'ln_icebergs', nemo_nl_file]) - if icb_rcode != 0: - sys.stderr.write('Unable to read ln_icebergs in &namberg namelist' - ' in the NEMO namelist file %s\n' - % nemo_nl_file) - sys.exit(error.SUBPROC_ERROR) - else: - if 'true' in icb_val.lower(): - return True - return False - - -def _verify_nemo_rst(cyclepointstr, nemo_rst, nemo_nl, nemo_nproc, nemo_version): - ''' - Verify that the full set of nemo restart files match. Currently this - is limited to the icebergs restart file. We require either a single - restart file, or a number of restart files equal to the number of - nemo processors. - ''' - restart_files = [f for f in os.listdir(nemo_rst) if - 'restart' in f] - - - if _get_ln_icebergs(nemo_nl): - - if nemo_version < 402: - # Pre nemo 4.2 compatibility - nemo_icb_regex = r'_icebergs_%s_restart(_\d+)?\.nc' % cyclepointstr - else: - # Post nemo 4.2 compatibility - nemo_icb_regex = r'_%s_restart_icb(_\d+)?\.nc' % cyclepointstr - - icb_restart_files = [f for f in restart_files if - re.findall(nemo_icb_regex, f)] - - # we can have a single rebuilt file, number of files equal to - # number of nemo processors, or rebuilt file and processor files. - if len(icb_restart_files) not in (1, nemo_nproc, nemo_nproc+1): - sys.stderr.write('[FAIL] Unable to find iceberg restart files for' - ' this cycle. Must either have one rebuilt file,' - ' as many as there are nemo processors (%i) or' - ' both rebuilt and processor files.' - '[FAIL] Found %i iceberg restart files\n' - % (nemo_nproc, len(icb_restart_files))) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - - -def _calc_current_model_date(model_basis_time, time_step, num_steps, - calendar): - ''' - Calculate the current model date using the basis time, - and the number of time-steps covered in a given model run. - - :arg: datetime model_basis_time : Model basis time - :arg: int time_step : Ocean model timestep (in seconds) - :arg: int num_steps : Num. timesteps covered in this - model run - :arg: string calendar : Calendar used in the model run - - ''' - ref_date_format = 'seconds since %Y-%m-%d %H:%M:%S' - - # modify the calendar names for compatability with cf_units module - if calendar == "360day": - calendar = "360_day" - if calendar == "365day": - calendar = "365_day" - - # Provide a reference time for the timestep incrementation. - ref_time = model_basis_time.strftime(ref_date_format) - - model_progress_secs = cf_units.date2num( - model_basis_time, ref_time, calendar=calendar) + (time_step * num_steps) - - current_model_date = cf_units.num2date( - model_progress_secs, ref_time, calendar=calendar) - - return current_model_date - - def _verify_fix_rst(restartdate, nemo_rst, model_basis_time, time_step, num_steps, calendar): ''' @@ -225,594 +97,296 @@ def _verify_fix_rst(restartdate, nemo_rst, model_basis_time, time_step, if fname_date > model_restart_date: common.remove_file(os.path.join(nemo_rst, restart_file)) restartdate = model_restart_date + return restartdate +def _calc_current_model_date(model_basis_time, time_step, num_steps, + calendar): + ''' + Calculate the current model date using the basis time, + and the number of time-steps covered in a given model run. + + :arg: datetime model_basis_time : Model basis time + :arg: int time_step : Ocean model timestep (in seconds) + :arg: int num_steps : Num. timesteps covered in this + model run + :arg: string calendar : Calendar used in the model run + + ''' + ref_date_format = 'seconds since %Y-%m-%d %H:%M:%S' + + # modify the calendar names for compatability with cf_units module + if calendar == "360day": + calendar = "360_day" + if calendar == "365day": + calendar = "365_day" + + # Provide a reference time for the timestep incrementation. + ref_time = model_basis_time.strftime(ref_date_format) + + model_progress_secs = cf_units.date2num( + model_basis_time, ref_time, calendar=calendar) + (time_step * num_steps) + + current_model_date = cf_units.num2date( + model_progress_secs, ref_time, calendar=calendar) + + return current_model_date -def _load_environment_variables(nemo_envar): +def _nemo_driver_assertions(nemo_envar): ''' - Load the NEMO environment variables required for the model run into the - nemo_envar container + Assertions prior to the running of the drivers, to ensure that we are + running in a parallel configuration, and the NEMO version is 3.6 or later ''' - # Load the nemo namelist environment variable - nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( - nemo_envar, dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL) - _ = _check_nemonl(nemo_envar) + assert int(nemo_envar['NEMO_VERSION']) >= 306, \ + '[FAIL] The python drivers are only valid for NEMO versions' \ + ' 3.6 and later.\n' - return nemo_envar + assert int(nemo_envar['NEMO_NPROC']) > 1, \ + '[FAIL] Nemo driver does not support the running of NEMO' \ + ' in serial mode\n' -def _setup_dates(common_env): +def _run_submodel_custom(common_env, nemo_envar, ln_restart, restart_ctl): + ''' + If the submodels have unique requirements we run those here + ''' + if 'top' in common_env['models']: + # ln_trcdta appears to always be the opposite of ln_restart, so we + # set it on that basis, though if this is correct it would seem to + # be redundant which should be addressed in the NEMO base code. + # These settings are based purely on the logic employed by the + # forerunner to this code in the MEDUSA-adapted UM10.6 GC3-coupled + # control script. + if ln_restart == ".true.": + ln_trcdta = ".false." + elif ln_restart == ".false.": + ln_trcdta = ".true." + else: + sys.stderr.write('[FAIL] TOP: invalid ln_restart value:' + ' %s\n' % ln_restart) + sys.exit(error.INVALID_LOCAL_ERROR) + # Update the TOP namelist + update_nemo_nl.update_top_nl(nemo_envar, ln_restart, restart_ctl, + ln_trcdta) + + if 'si3' in common_env['models']: + update_nemo_nl.update_si3_nl(nemo_envar) + +def _run_submodel_controllers(nemo_envar, common_env, restart_ctl, + nemo_dump_time, controller_mode): + ''' + Import and run the TOP/SI3 controllers if required + ''' + submodels = [] + # We check for the presence of lower case t, as in a lowered (TRUE, True, + # or true) + if 't' in nemo_envar['L_OCN_PASS_TRC'].lower(): + submodels.append('top') + sys.stdout.write('[INFO] nemo_driver: Passive tracer code is ' + 'active.\n') + else: + sys.stdout.write('[INFO] nemo_driver: ' + 'Passive tracer code not active.\n') + for submodel in submodels: + controller_name = '%s_controller' % submodel + sys.stdout.write('[INFO] Calling %s in %s mode\n' % + (controller_name, controller_mode)) + controller_mod = importlib.import_module(controller_name) + controller_mod.run_controller(common_env, + restart_ctl, + int(nemo_envar['NEMO_NPROC']), + common_env['RUNID'], + common_env['DRIVERS_VERIFY_RST'], + nemo_dump_time, + controller_mode) + +def _processor_restart_files_nrun(nemo_envar, common_env, nemo_init_dir): ''' - Setup the dates for the NEMO model run + Set up links to processor restart files for NRUN and return ln_restart ''' - calendar = common_env['CALENDAR'] - - sys.stdout.write('[INFO] NEMO calendar= %s ' % calendar) - if calendar == '360day': - calendar = '360' - nleapy = 30 - elif calendar == '365day': - calendar = '365' - nleapy = 0 - elif calendar == 'gregorian': - nleapy = 1 + nemo_warn = 'NEMO_START not set\nNEMO will use climatology\n' + ln_restart = ocn_lib.setup_nrun(nemo_envar['NEMO_START'], + nemo_envar['RST_LINK_DIR'], + 'restart', nemo_init_dir, + nemo_warn) + + # Set up Icebergs NRUN + icebergs_warn = 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n' + _ = ocn_lib.setup_nrun(nemo_envar['NEMO_ICEBERGS_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_icebergs', nemo_init_dir, + icebergs_warn) + + # Set up SI3 NRUN + if 'si3' in common_env['models']: + si3_warn = 'New SI3 run\n' + _ = ocn_lib.setup_nrun(nemo_envar['SI3_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_ice', nemo_init_dir, si3_warn) + + # Set up TOP NRUN + if 'top' in common_env['models']: + top_warn = 'New TOP run\n' + _ = ocn_lib.setup_nrun(nemo_envar['TOP_START'], + nemo_envar['RST_LINK_DIR'], + 'restart_trc', nemo_init_dir, top_warn) + return ln_restart + +def _processor_restart_files_crun(nemo_init_dir, restart_link_dir, + nemo_nproc, nemo_dump_time, runid): + ''' + Compile a list of relevant restart files for CRUNs + ''' + Restarts = collections.namedtuple( + 'Restarts', 'model file_label rst_label') + # Create a list of the various Restarts + restart_types = [ + Restarts('NEMO', '%so_%s_restart', 'restart'), + Restarts('SI3', '%so_%s_restart_ice', 'restart_ice'), + Restarts('Icebergs', '%so_icebergs_%s_restart', 'restart_icebergs'), + Restarts('TOP', '%so_%s_restart_trc', 'restart_trc')] + for i_restart in restart_types: + # Create links for restart files - either rebuilt or processor restarts + file_label = i_restart.file_label % (runid, nemo_dump_time) + ocn_lib.setup_restart(nemo_init_dir, restart_link_dir, + file_label, i_restart.rst_label, nemo_nproc, + i_restart.model) + +def _crun_timesteps(nemo_rst_date_bool, nemo_dump_time, nemo_step_int, + nemo_last_step, common_env): + '''Set up the timesteps for a CRUN''' + if not nemo_rst_date_bool: + # Nemo dump time is relative to the start of a model run and is an + # integer + nemo_dump_time = int(nemo_dump_time) + completed_days = nemo_dump_time * (nemo_step_int / 86400) + sys.stdout.write('[INFO] Nemo has previously completed %i days\n' % + completed_days) + ln_restart = ".true." + restart_ctl = 2 + if common_env['CONTINUE_FROM_FAIL'] == 'true': + # This is only used for coupled NWP where we don't have dates in + # NEMO restart file names + nemo_next_step = int(nemo_dump_time)+1 else: - sys.stderr.write('[FAIL] Calendar type %s not recognised\n' % - calendar) - sys.exit(error.INVALID_EVAR_ERROR) + nemo_next_step = nemo_last_step + 1 + return ln_restart, restart_ctl, nemo_next_step - #turn our times into lists of integers - model_basis = [int(i) for i in common_env['MODELBASIS'].split(',')] - run_start = [int(i) for i in common_env['TASKSTART'].split(',')] - run_length = [int(i) for i in common_env['TASKLENGTH'].split(',')] - run_days = inc_days.inc_days(run_start[0], run_start[1], run_start[2], - run_length[0], run_length[1], run_length[2], - calendar) - return nleapy, model_basis, run_start, run_length, run_days +def _nrun_timesteps(nemo_first_step): + '''Set up the timesteps for an NRUN''' + restart_ctl = 0 + nemo_next_step = nemo_first_step + nemo_last_step = nemo_first_step - 1 + return restart_ctl, nemo_next_step, nemo_last_step +def _verify_restart(common_env, nemo_envar, nemo_dump_time, nemo_rst): + ''' + Verify the dump time against cycle time if appropriate, do the + automatic fix, and check all other restart files match + ''' + if common_env['DRIVERS_VERIFY_RST'] == 'True': + nemo_dump_time = nemo_restart_lib.verify_fix_rst( + nemo_dump_time, + common_env['CYLC_TASK_CYCLE_POINT'], nemo_rst) + return nemo_dump_time def _setup_executable(common_env): ''' - Setup the environment and any files required by the executable + Set up the environment and any files required by the executable ''' - # Create the environment variable container - nemo_envar = dr_env_lib.env_lib.LoadEnvar() - # Load the environment variables required - nemo_envar = _load_environment_variables(nemo_envar) + # Load the environment variables required + nemo_envar, models = nemo_lib.load_environment_variables( + 'setup', common_env['models']) + common_env['models'] = models #Link the ocean executable common.remove_file(nemo_envar['OCEAN_LINK']) os.symlink(nemo_envar['OCEAN_EXEC'], nemo_envar['OCEAN_LINK']) - # Setup date variables + # Set up date variables nleapy, model_basis, run_start, \ - run_length, run_days = _setup_dates(common_env) - - # NEMO model setup - if int(nemo_envar['NEMO_VERSION']) < 306: - sys.stderr.write('[FAIL] The python drivers are only valid for nemo' - ' versions greater than 3.6') - sys.exit(error.INVALID_COMPONENT_VER_ERROR) - - # Read restart from nemo namelist - restart_direcs = [] - nemo_rst = _get_nemorst(nemo_envar['NEMO_NL']) - if nemo_rst: - restart_direcs.append(nemo_rst) - icerst_rcode, icerst_val = common.exec_subproc([ \ - 'grep', 'cn_icerst_dir', nemo_envar['NEMO_NL']]) - if icerst_rcode == 0: - ice_rst = re.findall(r'[\"\'](.*?)[\"\']', icerst_val)[0] - if ice_rst[-1] == '/': - ice_rst = ice_rst[:-1] - restart_direcs.append(ice_rst) - - for direc in restart_direcs: - # Strip white space - direc = direc.strip() - - # Check for trailing slashes in directory names and strip them - # out if they're present. - if direc.endswith('/'): - direc = direc.rstrip('/') - - if os.path.isdir(direc) and (direc not in ('./', '.')) and \ - common_env['CONTINUE'] == 'false': - sys.stdout.write('[INFO] directory is %s\n' % direc) - sys.stdout.write('[INFO] This is a New Run. Renaming old NEMO' - ' history directory\n') - - # In seasonal forecasting, we automatically apply - # short-stepping to re-try the model. Before re-attempting - # it, remove the associated NEMO history directory. - old_hist_dir = "%s.%s" % (direc, time.strftime("%Y%m%d%H%M")) - - if (common_env['SEASONAL'] == 'True' and - int(common_env['CYLC_TASK_TRY_NUMBER']) > 1): - common.remove_latest_hist_dir(old_hist_dir) - - os.rename(direc, old_hist_dir) - os.makedirs(direc) - elif not os.path.isdir(direc): - sys.stdout.write('[INFO] Creating NEMO restart directory:\n %s' % - direc) - os.makedirs(direc) - - # Compile a list of NEMO, seaice and iceberg restart files, if any exist. - # We look for files conforming to the naming convention: - # _yyyymmdd_restart_.nc OR - # _yyyymmdd_restart_icb_.nc where - # may itself contain underscores, hence we - # do not parse details based on counting the number of underscores. - nemo_restart_files = [f for f in os.listdir(nemo_rst) if - re.findall(r'.+_\d{8}_restart(_\d+)?\.nc', f) or - re.findall(r'.+_\d{8}_restart_icb(_\d+)?\.nc', f)] - nemo_restart_files.sort() - if nemo_restart_files: - latest_nemo_dump = nemo_rst + '/' + nemo_restart_files[-1] - else: - latest_nemo_dump = 'unset' - - nemo_init_dir = '.' - - # We need to ensure any lingering NEMO or iceberg retarts from - # previous runs are removed to ensure they're not accidentally - # picked up if we're starting from climatology on this occasion. - common.remove_file('restart.nc') - common.remove_file('restart_icebergs.nc') - common.remove_file('restart_icb.nc') - - if common_env['CONTINUE'] == 'false': - # This is a new run - sys.stdout.write('[INFO] New nemo run\n') - if os.path.isfile(latest_nemo_dump): - #os.path.isfile will return true for symbolic links aswell - sys.stdout.write('[INFO] Removing old NEMO restart data\n') - for file_path in glob.glob(nemo_rst+'/*restart*'): - common.remove_file(file_path) - for file_path in glob.glob(ice_rst+'/*restart*'): - common.remove_file(file_path) - for file_path in glob.glob(nemo_rst+'/*trajectory*'): - common.remove_file(file_path) - # source our history namelist file from the current directory in case - # of first cycle - history_nemo_nl = os.path.join(nemo_init_dir, nemo_envar['NEMO_NL']) - elif os.path.isfile(latest_nemo_dump): - sys.stdout.write('[INFO] Restart data available in NEMO restart ' - 'directory %s. Restarting from previous task output\n' - % nemo_rst) - sys.stdout.write('[INFO] Sourcing namelist file from the work ' - 'directory of the previous cycle\n') - # find the previous work directory if there is one - if common_env['CONTINUE_FROM_FAIL'] == 'false': - if common_env['CNWP_SUB_CYCLING'] == 'True': - prev_workdir = common.find_previous_workdir( \ - common_env['CYLC_TASK_CYCLE_POINT'], - common_env['CYLC_TASK_WORK_DIR'], - common_env['CYLC_TASK_NAME'], - common_env['CYLC_TASK_PARAM_run']) - else: - prev_workdir = common.find_previous_workdir( \ - common_env['CYLC_TASK_CYCLE_POINT'], - common_env['CYLC_TASK_WORK_DIR'], - common_env['CYLC_TASK_NAME']) - history_nemo_nl = os.path.join(prev_workdir, nemo_envar['NEMO_NL']) - else: - history_nemo_nl = nemo_envar['NEMO_NL'] - nemo_init_dir = nemo_rst - else: - sys.stderr.write('[FAIL] No restart data available in NEMO restart ' - 'directory:\n %s\n' % nemo_rst) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - - # Strings which are different pre-NEMO4.2 and at NEMO4.2 - if int(nemo_envar['NEMO_VERSION']) < 402: - gl_step_int_match = 'rn_rdt=' - iceberg_rst_part1 = '_icebergs_' - iceberg_rst_part2 = '_restart' - iceberg_link_name = 'restart_icebergs' - else: - gl_step_int_match = 'rn_dt=' - iceberg_rst_part1 = '_' - iceberg_rst_part2 = '_restart_icb' - iceberg_link_name = 'restart_icb' - - # Any variables containing things that can be globbed will start with gl_ - gl_first_step_match = 'nn_it000=' - gl_last_step_match = 'nn_itend=' - - gl_nemo_restart_date_match = 'ln_rstdate' - gl_model_basis_time = 'nn_date0=' - - # Read values from the nemo namelist file used by the previous cycle - # (if appropriate), or the configuration namelist if this is the initial - # cycle. - - # Make sure this file exists before trying to read it since restarted models - # may have had old work directories removed for numerous reasons. - if not os.path.isfile(history_nemo_nl): - sys.stderr.write('[FAIL] Cannot find namelist file %s to extract ' - 'timestep data.\n' % history_nemo_nl) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - - # First timestep of the previous cycle - _, first_step_val = common.exec_subproc(['grep', gl_first_step_match, - history_nemo_nl]) - - nemo_first_step = int(re.findall(r'.+=(.+),', first_step_val)[0]) - - # Last timestep of the previous cycle - _, last_step_val = common.exec_subproc(['grep', gl_last_step_match, - history_nemo_nl]) - nemo_last_step = re.findall(r'.+=(.+),', last_step_val)[0] - - # The string in the nemo time step field might have any one of - # a number of variants. e.g. "set_by_rose", "set_by_system", - # "set_by_um", etc. Hence we just check for the presence of - # purely numeric characters to see if we start from zero or not. - if nemo_last_step.isdigit(): - nemo_last_step = int(nemo_last_step) - else: - nemo_last_step = 0 - - # Determine (as an integer) the number of seconds per model timestep - _, nemo_step_int_val = common.exec_subproc(['grep', gl_step_int_match, - nemo_envar['NEMO_NL']]) - nemo_step_int = int(re.findall(r'.+=(\d*)', nemo_step_int_val)[0]) - - # If the value for nemo_rst_date_value is true then the model uses - # absolute date convention, otherwise the dump times are relative to the - # start of the model run and have an integer representation - _, nemo_rst_date_value = common.exec_subproc([ \ - 'grep', gl_nemo_restart_date_match, history_nemo_nl]) - if 'true' in nemo_rst_date_value: - nemo_rst_date_bool = True - else: - nemo_rst_date_bool = False + run_length, run_days = nemo_lib.setup_dates(common_env) + + # Make the required assertions + _nemo_driver_assertions(nemo_envar) + + # Read restart from current nemo namelists + nemo_rst, is_icebergs, ice_rst, top_rst = nemo_lib.read_current_cycle_nl( + common_env, nemo_envar) + + # we dont want any duplicates + restart_direcs = list( + collections.OrderedDict.fromkeys( + [direc for direc in [nemo_rst, ice_rst, top_rst] if direc])) + + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + + nemo_restart_files, latest_nemo_dump, nemo_init_dir \ + = nemo_restart_lib.compile_nemo_restart_files(nemo_rst) + + restart_nl_path = nemo_restart_lib.setup_previous_restart_nl( + common_env, nemo_rst, ice_rst, top_rst, latest_nemo_dump) + + history_nemo_nl = os.path.join(restart_nl_path, nemo_envar['NEMO_NL']) + + nemo_first_step, nemo_last_step, nemo_step_int, nemo_rst_date_bool \ + = nemo_lib.read_history_nl(history_nemo_nl) # The initial date of the model run (YYYYMMDD) nemo_ndate0 = '%04d%02d%02d' % tuple(model_basis[:3]) nemo_dump_time = "00000000" - # Get the model basis time for this run (YYYYMMDD) - _, model_basis_val = common.exec_subproc( - ['grep', gl_model_basis_time, history_nemo_nl]) - nemo_model_basis = re.findall(r'.+=(.+),', model_basis_val)[0] - if os.path.isfile(latest_nemo_dump): nemo_dump_time = re.findall(r'_(\d*)_restart', latest_nemo_dump)[0] - # Verify the dump time against cycle time if appropriate, do the - # automatic fix, and check all other restart files match - if common_env['DRIVERS_VERIFY_RST'] == 'True': - nemo_dump_time = _verify_fix_rst( - nemo_dump_time, - nemo_rst, - nemo_model_basis, nemo_step_int, nemo_last_step, - common_env['CALENDAR']) - - _verify_nemo_rst(nemo_dump_time, nemo_rst, nemo_envar['NEMO_NL'], - int(nemo_envar['NEMO_NPROC']), - int(nemo_envar['NEMO_VERSION'])) + nemo_dump_time = _verify_restart( + common_env, nemo_envar, nemo_dump_time, nemo_rst) # link restart files no that the last output one becomes next input one common.remove_file('restart.nc') - - common.remove_file('restart_ice_in.nc') + common.remove_file('restart_ice.nc') + common.remove_file('restart_trc.nc') # Sort out the processor restart files - if int(nemo_envar['NEMO_NPROC']) == 1: - sys.stderr.write('[FAIL] NEMO driver does not support the running' - ' of NEMO in serial mode\n') - sys.exit(SERIAL_MODE_ERROR) - else: + _processor_restart_files_crun( + nemo_init_dir, nemo_envar['RST_LINK_DIR'], + int(nemo_envar['NEMO_NPROC']), nemo_dump_time, common_env['RUNID']) - nemo_restart_count = 0 - ice_restart_count = 0 - iceberg_restart_count = 0 - - # Nemo has multiple processors - for i_proc in range(int(nemo_envar['NEMO_NPROC'])): - tag = str(i_proc).zfill(4) - nemo_rst_source = '%s/%so_%s_restart_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time, tag) - nemo_rst_link = 'restart_%s.nc' % tag - common.remove_file(nemo_rst_link) - - if os.path.isfile(nemo_rst_source): - os.symlink(nemo_rst_source, nemo_rst_link) - nemo_restart_count += 1 - - ice_rst_source = '%s/%so_%s_restart_ice_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time, tag) - if os.path.isfile(ice_rst_source): - ice_rst_link = 'restart_ice_in_%s.nc' % tag - common.remove_file(ice_rst_link) - os.symlink(ice_rst_source, ice_rst_link) - ice_restart_count += 1 - - iceberg_rst_source = '%s/%so%s%s%s_%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], iceberg_rst_part1, - nemo_dump_time, iceberg_rst_part2, tag) - if os.path.isfile(iceberg_rst_source): - iceberg_rst_link = '%s_%s.nc' % (iceberg_link_name, tag) - common.remove_file(iceberg_rst_link) - os.symlink(iceberg_rst_source, iceberg_rst_link) - iceberg_restart_count += 1 - #endfor - - if nemo_restart_count < 1: - sys.stdout.write('[INFO] No NEMO sub-PE restarts found\n') - # We found no nemo restart sub-domain files let's - # look for a global file. - nemo_rst_source = '%s/%so_%s_restart.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time) - if os.path.isfile(nemo_rst_source): - sys.stdout.write('[INFO] Using rebuilt NEMO restart '\ - 'file: %s\n' % nemo_rst_source) - nemo_rst_link = 'restart.nc' - common.remove_file(nemo_rst_link) - os.symlink(nemo_rst_source, nemo_rst_link) - - if ice_restart_count < 1: - sys.stdout.write('[INFO] No ice sub-PE restarts found\n') - # We found no ice restart sub-domain files let's - # look for a global file. - ice_rst_source = '%s/%so_%s_restart_ice.nc' % \ - (nemo_init_dir, common_env['RUNID'], \ - nemo_dump_time) - if os.path.isfile(ice_rst_source): - sys.stdout.write('[INFO] Using rebuilt ice restart '\ - 'file: %s\n' % ice_rst_source) - ice_rst_link = 'restart_ice_in.nc' - common.remove_file(ice_rst_link) - os.symlink(ice_rst_source, ice_rst_link) - - if iceberg_restart_count < 1: - sys.stdout.write('[INFO] No iceberg sub-PE restarts found\n') - # We found no iceberg restart sub-domain files let's - # look for a global file. - iceberg_rst_source = '%s/%so%s%s%s.nc' % \ - (nemo_init_dir, common_env['RUNID'], iceberg_rst_part1, - nemo_dump_time, iceberg_rst_part2) - if os.path.isfile(iceberg_rst_source): - sys.stdout.write('[INFO] Using rebuilt iceberg restart'\ - 'file: %s\n' % iceberg_rst_source) - iceberg_rst_nc = iceberg_link_name + '.nc' - common.remove_file(iceberg_rst_nc) - os.symlink(iceberg_rst_source, iceberg_rst_link) - - #endif (nemo_envar(NEMO_NPROC) == 1) - if nemo_rst_date_bool: - #Then nemo_dump_time has the form YYYYMMDD - pass - else: - #nemo_dump_time is relative to start of model run and is an - #integer - nemo_dump_time = int(nemo_dump_time) - completed_days = nemo_dump_time * (nemo_step_int // 86400) - sys.stdout.write('[INFO] Nemo has previously completed %i days\n' % - completed_days) - ln_restart = ".true." - restart_ctl = 2 - if common_env['CONTINUE_FROM_FAIL'] == 'true': - # This is only used for coupled NWP where we don't have dates in - # NEMO restart file names - nemo_next_step = int(nemo_dump_time)+1 - else: - nemo_next_step = nemo_last_step + 1 + ln_restart, restart_ctl, nemo_next_step \ + = _crun_timesteps(nemo_rst_date_bool, nemo_dump_time, nemo_step_int, + nemo_last_step, common_env) else: # This is an NRUN - if nemo_envar['NEMO_START'] != '': - if os.path.isfile(nemo_envar['NEMO_START']): - - os.symlink(nemo_envar['NEMO_START'], 'restart.nc') - ln_restart = ".true." - - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_START']): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_START']): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # a restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('restart_%s.nc' % proc_number) - - os.symlink(fname, 'restart_%s.nc' % proc_number) - ln_restart = ".true." - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_START'][:-3]): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_START'][:-3]): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # a restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('restart_%s.nc' % proc_number) - - os.symlink(fname, 'restart_%s.nc' % proc_number) - ln_restart = ".true." - else: - sys.stderr.write('[FAIL] file %s not found\n' % - nemo_envar['NEMO_START']) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - else: - #NEMO_START is unset - sys.stdout.write('[WARN] NEMO_START not set\n' - 'NEMO will use climatology\n') - ln_restart = ".false." - - if nemo_envar['NEMO_ICEBERGS_START'] != '': - - if os.path.isfile(nemo_envar['NEMO_ICEBERGS_START']): - - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - iceberg_rst_file = iceberg_link_name + '.nc' - common.remove_file(iceberg_rst_file) - os.symlink(nemo_envar['NEMO_ICEBERGS_START'], - iceberg_rst_file) - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_ICEBERGS_START']): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_ICEBERGS_START']): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('%s_%s.nc' % - (iceberg_rst_link, proc_number)) - - os.symlink(fname, '%s_%s.nc' % - (iceberg_rst_file, proc_number)) - elif os.path.isfile('%s_0000.nc' % - nemo_envar['NEMO_ICEBERGS_START'][:-3]): - for fname in glob.glob('%s_????.nc' % - nemo_envar['NEMO_ICEBERGS_START'][:-3]): - proc_number = fname.split('.')[-2][-4:] - - # We need to make sure there isn't already - # an iceberg restart file link set up, and if there is, get - # rid of it because symlink wont work otherwise! - common.remove_file('%s_%s.nc' % - (iceberg_rst_link, proc_number)) - - os.symlink(fname, '%s_%s.nc' % - (iceberg_rst_link, proc_number)) - else: - sys.stderr.write('[FAIL] file %s not found\n' % - nemo_envar['NEMO_ICEBERGS_START']) - sys.exit(error.MISSING_MODEL_FILE_ERROR) - else: - #NEMO_ICEBERGS_START unset - sys.stdout.write('[WARN] NEMO_ICEBERGS_START not set or file(s)' - ' not found. Icebergs (if switched on) will start' - ' from a state of zero icebergs\n') - restart_ctl = 0 - nemo_next_step = nemo_first_step - nemo_last_step = nemo_first_step - 1 + # Set up NEMO Nrun and ln_restart + ln_restart = _processor_restart_files_nrun(nemo_envar, common_env, + nemo_init_dir) + restart_ctl, nemo_next_step, nemo_last_step \ + = _nrun_timesteps(nemo_first_step) - if common_env['CONTINUE_FROM_FAIL'] == 'true': - #Check that the length of run is correct - #(it won't be if this is the wrong restart file) - run_start_dt = datetime.datetime(run_start[0], run_start[1], - run_start[2], run_start[3]) - model_basis_dt = datetime.datetime(model_basis[0], model_basis[1], - model_basis[2], model_basis[3]) - nemo_init_step = (run_start_dt-model_basis_dt).total_seconds() \ - /nemo_step_int - tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ - + run_length[4]*60 + run_length[5] - nemo_final_step = int((tot_runlen_sec//nemo_step_int) + nemo_init_step) - # Check that nemo_next_step is the correct number of hours to - # match LAST_DUMP_HOURS variable - steps_per_hour = 3600./nemo_step_int - last_dump_hrs = int(common_env['LAST_DUMP_HOURS']) - last_dump_step = int(nemo_init_step + last_dump_hrs*steps_per_hour) - if nemo_next_step-1 != last_dump_step: - sys.stderr.write('[FAIL] Last NEMO restarts not at correct time') - sys.exit(error.RESTART_FILE_ERROR) - else: - tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ - + run_length[4]*60 + run_length[5] - nemo_final_step = (tot_runlen_sec // nemo_step_int) + nemo_last_step - - - #Make our call to update the nemo namelist. First generate the list - #of commands - if int(nemo_envar['NEMO_VERSION']) >= 400: - # from NEMO 4.0 onwards we don't have jpnij in the namelist - update_nl_cmd = '--file %s --runid %so --restart %s --restart_ctl %s' \ - ' --next_step %i --final_step %s --start_date %s --leapyear %i' \ - ' --iproc %s --jproc %s --cpl_river_count %s --verbose' % \ - (nemo_envar['NEMO_NL'], \ - common_env['RUNID'], \ - ln_restart, \ - restart_ctl, \ - nemo_next_step, \ - nemo_final_step, \ - nemo_ndate0, \ - nleapy, \ - nemo_envar['NEMO_IPROC'], \ - nemo_envar['NEMO_JPROC'], \ - common_env['CPL_RIVER_COUNT']) - else: - update_nl_cmd = '--file %s --runid %so --restart %s --restart_ctl %s' \ - ' --next_step %i --final_step %s --start_date %s' \ - ' --leapyear %i --iproc %s --jproc %s --ijproc %s' \ - ' --cpl_river_count %s --verbose' % \ - (nemo_envar['NEMO_NL'], \ - common_env['RUNID'], \ - ln_restart, \ - restart_ctl, \ - nemo_next_step, \ - nemo_final_step, \ - nemo_ndate0, \ - nleapy, \ - nemo_envar['NEMO_IPROC'], \ - nemo_envar['NEMO_JPROC'], \ - nemo_envar['NEMO_NPROC'], \ - common_env['CPL_RIVER_COUNT']) - - update_nl_cmd = './update_nemo_nl %s' % update_nl_cmd - - # REFACTOR TO USE THE SAFE EXEC SUBPROC - update_nl_rcode, _ = common.__exec_subproc_true_shell([ \ - update_nl_cmd]) - if update_nl_rcode != 0: - sys.stderr.write('[FAIL] Error updating nemo namelist\n') - sys.exit(error.SUBPROC_ERROR) - - # We just check for the presence of T or t (as in TRUE, True or true) - # in the L_OCN_PASS_TRC value. - if ('T' in nemo_envar['L_OCN_PASS_TRC']) or \ - ('t' in nemo_envar['L_OCN_PASS_TRC']): + nemo_final_step = nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, run_days, + run_length, nemo_next_step, nemo_last_step) - sys.stdout.write('[INFO] nemo_driver: Passive tracer code is ' - 'active.\n') - controller_mode = "run_controller" + # Update the NEMO namelist + update_nemo_nl.update_nl(common_env, nemo_envar, ln_restart, restart_ctl, + nemo_next_step, nemo_final_step, nemo_ndate0, + nleapy) - top_controller.run_controller(common_env, - restart_ctl, - int(nemo_envar['NEMO_NPROC']), - common_env['RUNID'], - common_env['DRIVERS_VERIFY_RST'], - nemo_dump_time, - controller_mode) - else: - sys.stdout.write('[INFO] nemo_driver: ' - 'Passive tracer code not active\n.') - - use_si3 = 'si3' in common_env['models'] - if use_si3: - sys.stdout.write('[INFO] nemo_driver: SI3 code is active.\n') - controller_mode = "run_controller" - si3_controller.run_controller(common_env, - restart_ctl, - int(nemo_envar['NEMO_NPROC']), - common_env['RUNID'], - common_env['DRIVERS_VERIFY_RST'], - nemo_dump_time, - controller_mode) + # Deal with submodels that have unique requirements + _run_submodel_custom(common_env, nemo_envar, ln_restart, restart_ctl) return nemo_envar def _set_launcher_command(launcher, nemo_envar): ''' - Setup the launcher command for the executable + Set up the launcher command for the executable ''' if nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'] == 'unset': ss = False @@ -832,126 +406,32 @@ def _set_launcher_command(launcher, nemo_envar): nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'] return launch_cmd -def get_ocean_resol(nemo_nl_file, run_info): - ''' - Determine the ocean resolution. - This function is only used when creating the namcouple at run time. - ''' - # See if resolution is contained within namelists (existent of - # namelist_cfg has already been checked) - ocean_nml = f90nml.read(nemo_nl_file) - - # Check the required entries exist - if 'namcfg' not in ocean_nml: - sys.stderr.write('[FAIL] namcfg not found in namelist_cfg\n') - sys.exit(error.MISSING_OCN_RESOL_NML) - - if 'jpiglo' in ocean_nml['namcfg']: - # Resolution is contained within namelists - - if 'jpiglo' not in ocean_nml['namcfg'] or \ - 'jpjglo' not in ocean_nml['namcfg'] or \ - 'cp_cfg' not in ocean_nml['namcfg'] or \ - 'jp_cfg' not in ocean_nml['namcfg']: - sys.stderr.write('[FAIL] cp_cfg, jp_cfg, jpiglo or jpjglo are ' - 'missing from namelist namcf in namelist_cfg\n') - sys.exit(error.MISSING_OCN_RESOL) - - # Check it is on orca grid - if ocean_nml['namcfg']['cp_cfg'] != 'orca': - sys.stderr.write('[FAIL] we can currently only handle the ' - 'ORCA grid\n') - sys.exit(error.NOT_ORCA_GRID) - - # Check this is a grid we recognise - if ocean_nml['namcfg']['jp_cfg'] == 25: - run_info['OCN_grid'] = 'orca025' - else: - run_info['OCN_grid'] = 'orca' + str(ocean_nml['namcfg']['jp_cfg']) - - # Store the ocean resolution - run_info['OCN_resol'] = [ocean_nml['namcfg']['jpiglo'], - ocean_nml['namcfg']['jpjglo']] - - else: - # Resolution should be contained within a domain_cfg netCDF file. - # Rather than read this file, assume resolution is declared. - if 'OCN_grid' not in run_info: - sys.stderr.write('[FAIL] it is necessary to declare the ocean ' - 'resolution by setting the OCN_RES environment ' - 'variable.') - sys.exit(error.NOT_DECLARE_OCN_RES) - else: - # Determine the ocean resolution - if run_info['OCN_grid'] not in OCEAN_RESOLS: - sys.stderr.write('[FAIL] the ocean resolution for %s is ' - 'unknown' % run_info['OCN_grid']) - sys.exit(error.UNKNOWN_OCN_RESOL) - else: - run_info['OCN_resol'] = [OCEAN_RESOLS[run_info['OCN_grid']][0], - OCEAN_RESOLS[run_info['OCN_grid']][1]] - - return run_info - -def _sent_coupling_fields(nemo_envar, run_info): +def _write_output_file_to_stdout(out_file_path, error_count=0): ''' - Write the coupling fields sent from NEMO into model_snd_list. - This function is only used when creating the namcouple at run time. + Write ocean output file to standard out, and append the number of errors + to error_count (if present), or count and return number of errors in this + file if not present. The output files have unicode encoding, and + writing to standard out can't handle this, so we filter out any non ascii + characters from the line ''' - # Check that file specifying the coupling fields sent from - # NEMO is present - if not os.path.exists('OASIS_OCN_SEND'): - sys.stderr.write('[FAIL] OASIS_OCN_SEND is missing.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND) - - # Add toyatm to our list of executables - if not 'exec_list' in run_info: - run_info['exec_list'] = [] - run_info['exec_list'].append('toyoce') - - # Store ocean resolution if it is provided - if nemo_envar['OCN_RES']: - run_info['OCN_grid'] = nemo_envar['OCN_RES'] - - # Store the nemo version - if nemo_envar['NEMO_VERSION']: - run_info['NEMO_VERSION'] = nemo_envar['NEMO_VERSION'] - - # Determine the ocean resolution - run_info = get_ocean_resol(nemo_envar['NEMO_NL'], run_info) - - # If using the default coupling option, we'll need to read the - # NEMO namelist later - run_info['nemo_nl'] = nemo_envar['NEMO_NL'] - - # Read the namelist - oasis_nml = f90nml.read('OASIS_OCN_SEND') - - # Check we have the expected information - if 'oasis_ocn_send_nml' not in oasis_nml: - sys.stderr.write('[FAIL] namelist oasis_ocn_send_nml is ' - 'missing from OASIS_OCN_SEND.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND_NML) - if 'oasis_ocn_send' not in oasis_nml['oasis_ocn_send_nml']: - sys.stderr.write('[FAIL] entry oasis_ocn_send is missing ' - 'from namelist oasis_ocn_send_nml in ' - 'OASIS_OCN_SEND.\n') - sys.exit(error.MISSING_OASIS_OCN_SEND) - - # Create a list of fields sent from OCN - import write_namcouple - model_snd_list = \ - write_namcouple.add_to_cpl_list( \ - 'OCN', False, 0, - oasis_nml['oasis_ocn_send_nml']['oasis_ocn_send']) - - return run_info, model_snd_list - -def write_ocean_out_to_stdout(): + sys.stdout.write('[INFO] Ocean output from file %s\n' % out_file_path) + with open(out_file_path, 'r', encoding='utf8') as n_out: + for line in n_out: + # remove all non ascii characters + line = ''.join(i for i in line if ord(i) < 128) + sys.stdout.write(line) + if 'E R R O R' in line: + error_count += 1 + return error_count + + +def _write_ocean_out_to_stdout(): ''' - Write the contents of ocean.output to stnadard out + Write the contents of ocean.output to standard out, and determine if + there was an error in this run ''' + error_count = 0 # append the ocean output and solver stat file to standard out. Use an # iterator to read the files, incase they are too large to fit into # memory. Try to find both the NEMO 3.6 and NEMO 4.0 solver files for @@ -966,17 +446,23 @@ def write_ocean_out_to_stdout(): # The output file from NEMO4.0 has some suspect utf8 encoding, # this try/except will handle it if os.path.isfile(nemo_output_file): - sys.stdout.write('[INFO] Ocean output from file %s\n' % - nemo_output_file) - with open(nemo_output_file, 'r', encoding='utf-8') as n_out: - for line in n_out: - try: - sys.stdout.write(line) - except UnicodeEncodeError: - pass + error_count = _write_output_file_to_stdout( + nemo_output_file, error_count) else: sys.stdout.write('[INFO] Nemo output file %s not avaliable\n' % nemo_output_file) + return error_count + +def _copy_nl_end_of_run(nemo_envar_fin, nemo_rst, top_rst): + ''' + Copy namelist files at the end of run so next cycle can find them. We assume + nemo_rst will always exist + ''' + if os.path.isdir(nemo_rst): + shutil.copy(nemo_envar_fin['NEMO_NL'], nemo_rst) + if top_rst: + if os.path.isdir(top_rst): + shutil.copy(nemo_envar_fin['TOP_NL'], top_rst) def _finalize_executable(common_env): ''' @@ -987,10 +473,8 @@ def _finalize_executable(common_env): sys.stdout.write('[INFO] finalizing NEMO\n') sys.stdout.write('[INFO] running finalize in %s\n' % os.getcwd()) - write_ocean_out_to_stdout() + error_count = _write_ocean_out_to_stdout() - _, error_count = common.__exec_subproc_true_shell([ \ - 'grep "E R R O R" ocean.output | wc -l']) if int(error_count) >= 1: sys.stderr.write('[FAIL] An error has been found with the NEMO run.' ' Please investigate the ocean.output file for more' @@ -999,31 +483,12 @@ def _finalize_executable(common_env): # move the nemo namelist to the restart directory to allow the next cycle # to pick it up - nemo_envar_fin = dr_env_lib.env_lib.LoadEnvar() - nemo_envar_fin = dr_env_lib.env_lib.load_envar_from_definition( - nemo_envar_fin, dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL) - nemo_rst = _get_nemorst(nemo_envar_fin['NEMO_NL']) - if os.path.isdir(nemo_rst) and \ - os.path.isfile(nemo_envar_fin['NEMO_NL']): - shutil.copy(nemo_envar_fin['NEMO_NL'], nemo_rst) - - # The only way to check if TOP is active is by checking the - # passive tracer env var. - - # Check whether we need to finalize the TOP controller - if ('T' in nemo_envar_fin['L_OCN_PASS_TRC']) or \ - ('t' in nemo_envar_fin['L_OCN_PASS_TRC']): - - sys.stdout.write('[INFO] nemo_driver: Finalize TOP controller.') - - controller_mode = "finalize" - top_controller.run_controller([], [], [], [], [], [], controller_mode) - - use_si3 = 'si3' in common_env['models'] - if use_si3: - sys.stdout.write('[INFO] nemo_driver: Finalise SI3 controller\n') - controller_mode = "finalize" - si3_controller.run_controller([], [], [], [], [], [], controller_mode) + nemo_envar_fin, models = nemo_lib.load_environment_variables( + 'final', common_env['models']) + common_env['models'] = models + nemo_rst, _, _, top_rst = nemo_lib.read_current_cycle_nl( + common_env, nemo_envar_fin) + _copy_nl_end_of_run(nemo_envar_fin, nemo_rst, top_rst) def run_driver(common_env, mode, run_info): ''' @@ -1032,13 +497,13 @@ def run_driver(common_env, mode, run_info): ''' if mode == 'run_driver': exe_envar = _setup_executable(common_env) - launch_cmd = _set_launcher_command(common_env['ROSE_LAUNCHER'], - exe_envar) + launch_cmd = _set_launcher_command(common_env['ROSE_LAUNCHER'], exe_envar) if run_info['l_namcouple']: model_snd_list = None else: run_info, model_snd_list = \ - _sent_coupling_fields(exe_envar, run_info) + nemo_runtime_namcouple.sent_coupling_fields( + exe_envar, run_info) elif mode == 'finalize': _finalize_executable(common_env) exe_envar = None @@ -1046,7 +511,7 @@ def run_driver(common_env, mode, run_info): model_snd_list = None elif mode == 'failure': # subset of operations of the model fails - write_ocean_out_to_stdout() + _write_ocean_out_to_stdout() exe_envar = None launch_cmd = None model_snd_list = None diff --git a/Coupled_Drivers/nemo_lib.py b/Coupled_Drivers/nemo_lib.py new file mode 100644 index 00000000..38c0d9bf --- /dev/null +++ b/Coupled_Drivers/nemo_lib.py @@ -0,0 +1,259 @@ +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_lib.py + +DESCRIPTION + Library functions required for the NEMO driver +''' +import datetime +import os +import sys + +import common +import dr_env_lib.env_lib +import dr_env_lib.nemo_def +import dr_env_lib.ocn_cont_def +import error +import inc_days +import ocn_lib + +# +# SECTION ONE: +# +# Section one contains functions that can be called directly from other +# modules +# + +def read_current_cycle_nl(common_env, nemo_envar): + ''' + Read the current cycle namelist for each item in turn, NEMO, SI3 and TOP + ''' + nemo_rst, ln_icebergs, ice_rst, top_rst = None, None, None, None + if 'nemo' in common_env['models']: + nemo_rst, ln_icebergs = _read_current_cycle_nl_nemo( + nemo_envar['NEMO_NL']) + if 'si3' in common_env['models']: + ice_rst = _read_current_cycle_nl_si3(nemo_envar['SI3_NL']) + if 'top' in common_env['models']: + top_rst = _read_current_cycle_nl_top(nemo_envar['TOP_NL']) + return nemo_rst, ln_icebergs, ice_rst, top_rst + + + +def setup_dates(common_env): + ''' + Setup the dates for the NEMO model run + ''' + calendar = common_env['CALENDAR'] + if calendar == '360day': + calendar = '360' + nleapy = 30 + elif calendar == '365day': + calendar = '365' + nleapy = 0 + elif calendar == 'gregorian': + nleapy = 1 + else: + sys.stderr.write('[FAIL] Calendar type %s not recognised\n' % + calendar) + sys.exit(error.INVALID_EVAR_ERROR) + + #turn our times into lists of integers + model_basis = [int(i) for i in common_env['MODELBASIS'].split(',')] + run_start = [int(i) for i in common_env['TASKSTART'].split(',')] + run_length = [int(i) for i in common_env['TASKLENGTH'].split(',')] + + run_days = inc_days.inc_days(run_start[0], run_start[1], run_start[2], + run_length[0], run_length[1], run_length[2], + calendar) + return nleapy, model_basis, run_start, run_length, run_days + +def read_history_nl(history_nemo_nl): + ''' + Read and process the desired variables from the history nemo namelist + ''' + namelist = ocn_lib.ReadOcnNamelist(history_nemo_nl) + nemo_first_step, nemo_last_step, nemo_step_int, nemo_rst_date \ + = namelist.read_variables( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date') + return (int(nemo_first_step), _check_nemo_last_step(nemo_last_step), + int(nemo_step_int), string_to_boolean(nemo_rst_date)) + +def load_environment_variables(mode, models): + ''' + Load the environment variables for NEMO and SI3 into a single + container + ''' + nemo_envar = dr_env_lib.env_lib.LoadEnvar() + # The functions to check the namelists for each model + checks = {'nemo': _check_nemonl, + 'si3': _check_si3nl, + 'top': _check_topnl} + if mode == 'setup': + defs = { + 'nemo': dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL, + 'si3': dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_INITIAL, + 'top': dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_INITIAL} + elif mode == 'final': + defs = { + 'nemo': dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL, + 'si3': dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_FINAL, + 'top': dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_FINAL} + for imodel in models.split(' '): + if imodel in defs.keys(): + nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( + nemo_envar, defs[imodel]) + checks[imodel](nemo_envar) + # TOP is controlled by the environment variable 'L_OCN_PASS_TRC' rather + # than in the models list, we must add the environment variables + if 't' in nemo_envar['L_OCN_PASS_TRC'].lower(): + models = models + ' top' + nemo_envar = dr_env_lib.env_lib.load_envar_from_definition( + nemo_envar, defs['top']) + checks['top'](nemo_envar) + return nemo_envar, models + +def string_to_boolean(nemo_rst_date): + ''' + If a string contains a string containg upper or lower case true + (or some permeatation thereof), return python boolean True. Otherwise + return False + ''' + if 'true' in nemo_rst_date.lower(): + return True + return False + + +def setup_nemo_runlen(common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, nemo_last_step): + ''' + Setup the nemo runlength for coupled NWP models with subcycle restarts + ''' + if common_env['CONTINUE_FROM_FAIL'] == 'true': + # Check the length of the run is correct + # (it won't be if this is the wrong restart file) + run_start_dt = datetime.datetime(run_start[0], run_start[1], + run_start[2], run_start[3]) + model_basis_dt = datetime.datetime(model_basis[0], model_basis[1], + model_basis[2], model_basis[3]) + nemo_init_step = (run_start_dt-model_basis_dt).total_seconds() \ + /nemo_step_int + tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ + + run_length[4]*60 + run_length[5] + nemo_final_step = int((tot_runlen_sec//nemo_step_int) + nemo_init_step) + # Check that nemo_next_step is the correct number of hours to + # match LAST_DUMP_HOURS variable + steps_per_hour = 3600./nemo_step_int + last_dump_hrs = int(common_env['LAST_DUMP_HOURS']) + last_dump_step = int(nemo_init_step + last_dump_hrs*steps_per_hour) + if nemo_next_step-1 != last_dump_step: + sys.stderr.write('[FAIL] Last NEMO restarts not at correct time\n' + ' Last completed timestep %d\n' + ' Expected next step %d\n' + ' Actual next step %d\n' + % (last_dump_step, last_dump_step+1, + nemo_next_step)) + sys.exit(error.RESTART_FILE_ERROR) + else: + tot_runlen_sec = run_days * 86400 + run_length[3]*3600 \ + + run_length[4]*60 + run_length[5] + nemo_final_step = (tot_runlen_sec // nemo_step_int) + nemo_last_step + return nemo_final_step + + +# +# SECTION TWO: +# +# Section two contains functions that are private to this module +# + +def _check_nemo_last_step(nemo_last_step): + ''' + The string in the nemo time step field might have any one of + a number of variants. e.g. "set_by_rose", "set_by_system", + "set_by_um", etc. Hence we just check for the presence of + purely numeric characters to see if we start from zero or not. + ''' + if nemo_last_step.isdigit(): + return int(nemo_last_step) + return 0 + +def _check_nemonl(envar_container): + ''' + As the environment variable NEMO_NL is required by both the setup + and finalise functions, this will be encapsulated here + ''' + # Information will be retrieved from this file during the running of the + # driver, so check it exists + if not os.path.isfile(envar_container['NEMO_NL']): + sys.stderr.write('[FAIL] Can not find the nemo namelist file %s\n' % + envar_container['NEMO_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + +def _check_si3nl(envar_container): + ''' + Check the si3 namelist file exists as information will be retrieved from + this file during the running of the driver + ''' + if not os.path.isfile(envar_container['SI3_NL']): + sys.stderr.write('[FAIL] Can not find the SI3 namelist ' + 'file %s\n' % envar_container['SI3_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + +def _check_topnl(envar_container): + ''' + Check the TOP namelist file exists as information will be retrieved from + this file during the running of the driver''' + if not os.path.isfile(envar_container['TOP_NL']): + sys.stderr.write('[FAIL] Can not find the TOP namelist ' + 'file %s\n' % envar_container['TOP_NL']) + sys.exit(error.MISSING_DRIVER_FILE_ERROR) + + +def _read_current_cycle_nl_nemo(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for NEMO + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + nemo_rst, ln_icebergs = current_nl.read_variables( + 'nemo_rst', 'ln_icebergs') + if nemo_rst: + nemo_rst = common.remove_trailing_slash(nemo_rst) + return nemo_rst, string_to_boolean(ln_icebergs) + +def _read_current_cycle_nl_si3(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for SI3 + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + ice_rst = current_nl.read_variables('ice_rst') + if ice_rst: + ice_rst = common.remove_trailing_slash(ice_rst) + sys.stdout.write('[INFO] NEMO running with SI3 submodel\n') + return ice_rst + +def _read_current_cycle_nl_top(namelist_path): + ''' + Read the variables required from the current cycle namelist and + modify as required for TOP + ''' + current_nl = ocn_lib.ReadOcnNamelist(namelist_path) + top_rst = current_nl.read_variables('top_rst') + if top_rst: + top_rst = common.remove_trailing_slash(top_rst) + sys.stdout.write('[INFO] NEMO running with TOP submodel\n') + return top_rst diff --git a/Coupled_Drivers/nemo_restart_lib.py b/Coupled_Drivers/nemo_restart_lib.py new file mode 100644 index 00000000..84969643 --- /dev/null +++ b/Coupled_Drivers/nemo_restart_lib.py @@ -0,0 +1,181 @@ +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_restart_lib.py + +DESCRIPTION + Library functions required to control the starting or restarting of + NEMO ocean model +''' +import glob +import os +import re +import sys +import time + +import common +import error +# +# SECTION ONE: +# +# Section one contains functions that can be called directly from other +# modules +# + +def setup_previous_restart_nl( + common_env, nemo_rst, ice_rst, top_rst, latest_nemo_dump): + ''' + Setup the path to the previous restart namelist if required + ''' + if common_env['CONTINUE'] == 'false': + sys.stdout.write('[INFO] New nemo run\n') + _setup_previous_restart_nl_nrun(nemo_rst, ice_rst, top_rst) + # The restart is in the current working directory in this case + return '.' + if os.path.isfile(latest_nemo_dump): + sys.stdout.write('[INFO] Restart data available in NEMO restart ' + 'directory %s. Restarting from previous task output\n' + '[INFO] Sourcing namelist file from the work ' + 'directory of the previous cycle\n' + % nemo_rst) + # find the previous work directory if there is one + return _setup_previous_restart_nl_crun(common_env) + sys.stderr.write('[FAIL] No restart data available in NEMO restart ' + 'directory:\n %s\n' % nemo_rst) + sys.exit(error.MISSING_MODEL_FILE_ERROR) + +def verify_fix_rst(restartdate, cyclepoint, nemo_rst): + ''' + Verify that the restart file for nemo is at the cyclepoint for the + start of this cycle. The cyclepoint variable has form + yyyymmddThhmmZ, restart date yyyymmdd. If they don't match, then + make sure that nemo restarts from the correct restart date + ''' + cycle_date_string = cyclepoint.split('T')[0] + if restartdate == cycle_date_string: + sys.stdout.write('[INFO] Validated NEMO restart date\n') + else: + # Write the message to both standard out and standard error + msg = '[WARN] The NEMO restart data does not match the ' \ + ' current cycle time\n.' \ + ' Cycle time is %s\n' \ + ' NEMO restart time is %s\n' \ + '[WARN] Automatically removing NEMO dumps ahead of ' \ + 'the current cycletime, and pick up the dump at ' \ + 'this time\n' % (cycle_date_string, restartdate) + sys.stdout.write(msg) + sys.stderr.write(msg) + #Remove all nemo restart files that are later than the correct + #cycle times + #Make our generic restart regular expression, to cover normal NEMO + #restart, and potential iceberg or passive tracer restart files, for + #both the rebuilt and non rebuilt cases + generic_rst_regex = r'(icebergs)?.*restart(_trc)?(_\d+)?\.nc' + all_restart_files = [f for f in os.listdir(nemo_rst) if + re.findall(generic_rst_regex, f)] + for restart_file in all_restart_files: + fname_date = re.findall(r'\d{8}', restart_file)[0] + if fname_date > cycle_date_string: + common.remove_file(os.path.join(nemo_rst, restart_file)) + restartdate = cycle_date_string + return restartdate + +def compile_nemo_restart_files(nemo_rst): + ''' + Compile a list of the NEMO restart files, if any exist. + We look for files conforming to the naming convention: + _yyyymmdd_restart_.nc where + may itself contain underscores, hence we + do not parse details based on counting the number of underscores. + ''' + nemo_restart_files = [f for f in os.listdir(nemo_rst) if + re.findall(r'.+_\d{8}_restart(_\d+)?\.nc', f)] + nemo_restart_files.sort() + if nemo_restart_files: + latest_nemo_dump = os.path.join(nemo_rst, nemo_restart_files[1]) + else: + latest_nemo_dump = 'unset' + # We need to ensure any lingering NEMO or iceberg retarts from + # previous runs are removed to ensure they're not accidentally + # picked up if we're starting from climatology on this occasion. + common.remove_file('restart.nc') + common.remove_file('restart_icebergs.nc') + common.remove_file('restart_trc.nc') + + if os.path.isfile(latest_nemo_dump): + nemo_init_dir = nemo_rst + else: + nemo_init_dir = '.' + return nemo_restart_files, latest_nemo_dump, nemo_init_dir + +def create_restart_direcs(restart_direcs, common_env): + ''' + Take a list of restart directorys. For an NRUN archive any existing + and create new ones + ''' + if common_env['CONTINUE'] == 'false': + # We only need to manipulate restart directories for NRUNS + # Create the time string here so that all directories will have the + # same time + time_str = time.strftime("%Y%m%d%H%M") + for direc in restart_direcs: + isdir = os.path.isdir(direc) + if isdir and (direc not in ('./', '.')): + sys.stdout.write('[INFO] directory is %s\n' % direc) + sys.stdout.write('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n') + os.rename(direc, '%s.%s' % (direc, time_str)) + os.makedirs(direc) + elif not isdir: + sys.stdout.write('[INFO] Creating NEMO restart directory:\n' + ' %s\n' % direc) + os.makedirs(direc) + + +# +# SECTION TWO: +# +# Section two contains functions that are private to this module +# + +def _setup_previous_restart_nl_nrun(nemo_rst, ice_rst, top_rst): + ''' + Delete the restarts when starting an NRUN + ''' + restart_groups = [nemo_rst+'/*restart*', nemo_rst+'/*trajectory*'] + if ice_rst: + restart_groups.append(ice_rst+'/*restart*') + if top_rst: + restart_groups.append(top_rst+'/*restart*') + for glob_exp in restart_groups: + for file_path in glob.glob(glob_exp): + common.remove_file(file_path) + +def _setup_previous_restart_nl_crun(common_env): + ''' + Setup directories and path to nemo namelist for CRUNS + ''' + if common_env['CONTINUE_FROM_FAIL'] == 'false': + if common_env['CNWP_SUB_CYCLING'] == 'True': + prev_workdir = common.find_previous_workdir(\ + common_env['CYLC_TASK_CYCLE_POINT'], + common_env['CYLC_TASK_WORK_DIR'], + common_env['CYLC_TASK_NAME'], + common_env['CYLC_TASK_PARAM_run']) + else: + prev_workdir = common.find_previous_workdir( \ + common_env['CYLC_TASK_CYCLE_POINT'], + common_env['CYLC_TASK_WORK_DIR'], + common_env['CYLC_TASK_NAME']) + return prev_workdir + return '' diff --git a/Coupled_Drivers/nemo_runtime_namcouple.py b/Coupled_Drivers/nemo_runtime_namcouple.py new file mode 100644 index 00000000..a953c455 --- /dev/null +++ b/Coupled_Drivers/nemo_runtime_namcouple.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + nemo_runtime_namcouple.py + +DESCRIPTION + Library of function required for the creation of the NEMO namcouple + components at runtime +''' +import os +import sys +import write_namcouple +import error +try: + import f90nml +except ImportError: + pass + +def get_ocean_resol(nemo_nl_file, run_info): + ''' + Determine the ocean resolution. + This function is only used when creating the namcouple at run time. + ''' + + # Read in the resolution of ocean (existent of namelist_cfg has + # already been checked) + ocean_nml = f90nml.read(nemo_nl_file) + + # Check the required entries exist + if 'namcfg' not in ocean_nml: + sys.stderr.write('[FAIL] namcfg not found in namelist_cfg\n') + sys.exit(error.MISSING_OCN_RESOL_NML) + if 'jpiglo' not in ocean_nml['namcfg'] or \ + 'jpjglo' not in ocean_nml['namcfg'] or \ + 'cp_cfg' not in ocean_nml['namcfg'] or \ + 'jp_cfg' not in ocean_nml['namcfg']: + sys.stderr.write('[FAIL] cp_cfg, jp_cfg, jpiglo or jpjglo are ' + 'missing from namelist namcf in namelist_cfg\n') + sys.exit(error.MISSING_OCN_RESOL) + + # Check it is on orca grid + if ocean_nml['namcfg']['cp_cfg'] != 'orca': + sys.stderr.write('[FAIL] we can currently only handle the ' + 'ORCA grid\n') + sys.exit(error.NOT_ORCA_GRID) + + # Check this is a grid we recognise + if ocean_nml['namcfg']['jp_cfg'] == 25: + run_info['OCN_grid'] = 'orca025' + else: + run_info['OCN_grid'] = 'orca' + str(ocean_nml['namcfg']['jp_cfg']) + + # Store the ocean resolution + run_info['OCN_resol'] = [ocean_nml['namcfg']['jpiglo'], + ocean_nml['namcfg']['jpjglo']] + + return run_info + +def sent_coupling_fields(nemo_envar, run_info): + ''' + Write the coupling fields sent from NEMO into model_snd_list. + This function is only used when creating the namcouple at run time. + ''' + # Check that file specifying the coupling fields sent from + # NEMO is present + if not os.path.exists('OASIS_OCN_SEND'): + sys.stderr.write('[FAIL] OASIS_OCN_SEND is missing.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND) + + # Add toyatm to our list of executables + if not 'exec_list' in run_info: + run_info['exec_list'] = [] + run_info['exec_list'].append('toyoce') + + # Determine the ocean resolution + run_info = get_ocean_resol(nemo_envar['NEMO_NL'], run_info) + + # If using the default coupling option, we'll need to read the + # NEMO namelist later + run_info['nemo_nl'] = nemo_envar['NEMO_NL'] + + # Read the namelist + oasis_nml = f90nml.read('OASIS_OCN_SEND') + + # Check we have the expected information + if 'oasis_ocn_send_nml' not in oasis_nml: + sys.stderr.write('[FAIL] namelist oasis_ocn_send_nml is ' + 'missing from OASIS_OCN_SEND.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND_NML) + if 'oasis_ocn_send' not in oasis_nml['oasis_ocn_send_nml']: + sys.stderr.write('[FAIL] entry oasis_ocn_send is missing ' + 'from namelist oasis_ocn_send_nml in ' + 'OASIS_OCN_SEND.\n') + sys.exit(error.MISSING_OASIS_OCN_SEND) + + # Create a list of fields sent from OCN + model_snd_list = \ + write_namcouple.add_to_cpl_list( \ + 'OCN', False, 0, + oasis_nml['oasis_ocn_send_nml']['oasis_ocn_send']) + + return run_info, model_snd_list diff --git a/Coupled_Drivers/ocn_lib.py b/Coupled_Drivers/ocn_lib.py new file mode 100644 index 00000000..c44cc6fe --- /dev/null +++ b/Coupled_Drivers/ocn_lib.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + ocn_lib.py + +DESCRIPTION + Library of function required for the ocean models +''' +from collections import namedtuple +import glob +import os +import re +import sys + +import common +import error + + +class ReadOcnNamelist: + ''' + A class to define the behaviours for reading from a wet model namelist. + We define the potential variables and their regular expressions to read + from the namelist file + ''' + def __init__(self, namelist_file): + ''' + Constructor for the object to read from given namelist file + ''' + self._to_read = {} + self.namelist_file = namelist_file + NamelistVal = namedtuple('NamelistVal', 'regex value') + self._variables \ + = {'nemo_first_step': NamelistVal(r'nn_it000=(.+),', None), + 'nemo_last_step': NamelistVal(r'nn_itend=(.+),', None), + 'nemo_step_int': NamelistVal(r'rn_rdt=(\d*)', None), + 'nemo_rst_date': NamelistVal(r'ln_rstdate=(.+),', None), + 'nemo_rst': NamelistVal( + r'cn_ocerst_outdir=[\"\'](.*?)[\"\']', None), + 'ice_rst': NamelistVal( + r'cn_icerst_outdir=[\"\'](.*?)[\"\']', None), + 'ln_icebergs': NamelistVal(r'ln_icebergs=(.+),', None), + 'top_rst': NamelistVal( + r'cn_trcrst_outdir=[\"\'](.*?)[\"\']', None)} + + def read_variables(self, *requested_variables): + ''' + Read an arbitary number of variable names and return their values. + If the variable doesnt exist, return none + ''' + # Reset _to_read + self._to_read = {} + for requested_var in requested_variables: + if requested_var not in self._variables.keys(): + sys.stderr.write('[FAIL] Unable to determine how to read the' + ' variable %s from file %s\n' % + (requested_var, self.namelist_file)) + sys.exit(error.MISSING_FILE_CONTENTS_ERROR) + self._to_read[requested_var] = self._variables[requested_var] + self._read_file() + # loop again to get the output in the correct order + return_list = [] + for requested_var in requested_variables: + return_list.append(self._to_read[requested_var].value) + if len(return_list) == 1: + return return_list[0] + return tuple(return_list) + + def _read_file(self): + '''Read the variables from the namelist file''' + with open(self.namelist_file, 'r') as nl_fh: + for line in nl_fh.readlines(): + for var in self._to_read.keys(): + try: + self._to_read[var] \ + = self._to_read[var]._replace(value=(re.findall( + self._to_read[var].regex, line)[0])) + except IndexError: + pass + + + +# +# Functions to interface to the library +# + +def setup_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, model_dscr): + ''' + Setup wet model restart file links (NEMO, TOP, SI3) + ''' + _create_restart_link_dir(restart_link_dir) + nproc_str_len = _get_nemo_nproc_str(init_dir) + if nproc_str_len > 0: + _setup_multiple_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, nproc_str_len) + else: + # It is likely we might have a rebuilt restart file + _setup_single_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, model_dscr) + + +def setup_nrun(start_envar, restart_link_dir, restart_link_label, init_dir, + warning_message): + ''' + Setup restart files for an NRUN + ''' + _create_restart_link_dir(restart_link_dir) + if start_envar: + len_proc_num = _get_nemo_nproc_str(init_dir) + if os.path.isfile(start_envar): + restart_link_label = '%s.nc' % restart_link_label + os.symlink(start_envar, + os.path.join(restart_link_dir, restart_link_label)) + ln_restart = ".true." + elif os.path.isfile('%s_%s.nc' % + (start_envar, '0'*len_proc_num)): + for fname in glob.glob('%s_%s.nc' % + (start_envar, '?'*len_proc_num)): + proc_number = fname.split('.')[-2][-len_proc_num:] + restart_link_file = os.path.join( + restart_link_dir, '%s_%s.nc' % (restart_link_label, + proc_number)) + common.remove_file(restart_link_file) + os.symlink(fname, restart_link_file) + ln_restart = ".true." + else: + sys.stderr.write('[FAIL] file %s not found\n' % (start_envar)) + sys.exit(error.MISSING_MODEL_FILE_ERROR) + else: + # Start envar is unset + sys.stdout.write('[WARN] %s\n' % warning_message) + ln_restart = ".false." + return ln_restart + +# +# Functions to manipulate restart file links +# + +def _get_nemo_nproc_str(restart_file_dir='.'): + ''' + Get the length of the string in the nemo restart file name containing + the processor number for unbuilt restart files. For example + restart_0012.nc would return 4. Takes in a directory to search for + the restart file. If no directory specified we assume it is in the + current directory + ''' + # Find how large the processor tag is on the restart files + # Get the zero tagged NEMO restart file + rst_file_glob = '%s/*_[0-9]*_restart_*0.nc' % restart_file_dir + try: + zero_nemo_rst = glob.glob(rst_file_glob)[0] + print(zero_nemo_rst) + # Get the length of the strings of zeros + len_proc_num = len(re.search(r'.*_(\d+).nc', zero_nemo_rst).group(1)) + except IndexError: + len_proc_num = 0 + return len_proc_num + + +def _setup_multiple_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, nemo_nproc, nproc_str_len): + ''' + Setup links for multiple (unrebuilt) restart files + ''' + for i_proc in range(nemo_nproc): + tag = str(i_proc).zfill(nproc_str_len) + rst_source = '%s/%s_%s.nc' % (init_dir, + file_name_label, + tag) + rst_link = os.path.join(restart_link_dir, + '%s_%s.nc' % (restart_link_label, tag)) + common.remove_file(rst_link) + + if os.path.isfile(rst_source): + os.symlink(rst_source, rst_link) + +def _setup_single_restart(init_dir, restart_link_dir, file_name_label, + restart_link_label, model_dscr): + ''' + Setup links for single (rebuilt) restart file + ''' + sys.stdout.write('[INFO] No %s sub-PE restarts found\n' % model_dscr) + rst_source = '%s/%s.nc' % (init_dir, file_name_label) + rst_link = os.path.join(restart_link_dir, + '%s.nc' % restart_link_label) + common.remove_file(rst_link) + if os.path.isfile(rst_source): + sys.stdout.write('[INFO] Using rebuilt %s restart file %s\n' + % (model_dscr, rst_source)) + os.symlink(rst_source, rst_link) + +def _create_restart_link_dir(dirname): + ''' + Try to create the restart link directory, if it already exists we carry + on as normal''' + if dirname != '.': + try: + os.mkdir(dirname) + except FileExistsError: + sys.stdout.write('[INFO] Restart link subdirectory %s exists\n' % + dirname) diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 4e4b690c..4ee47e73 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -1,6 +1,6 @@ #!/usr/bin/env bash #*****************************COPYRIGHT****************************** -# (C) Crown copyright 2021-2025 Met Office. All rights reserved. +# (C) Crown copyright 2021-2026 Met Office. All rights reserved. # # Use, duplication or disclosure of this code is subject to the restrictions # as set forth in the licence. If no licence has been raised with this copy @@ -27,7 +27,7 @@ echo $CYLC_SUITE_RUN_DIR # compatibility, and future flexibility DRIVER_EXTRACT_DIR=${DRIVER_EXTRACT_DIR:=$CYLC_SUITE_SHARE_DIR/fcm_make_drivers} -echo "[DRIVER_TEST_SCRIPT] DRIVER_EXTRACT_DIR is $DRIVER_EXTRACT_DIR" +echo "[DRIVER_TEST_SCRIPT] DRIVER_EXTRACT_DIR is $DRIVER_EXTRACT_DIR" # This lets us set up a failure mode export DRIVERS_FINALISED=false @@ -74,6 +74,9 @@ if [[ ${EXTRA_LINK_DRIVERS_INFO:-} == "true" ]]; then lfs getstripe ./link_drivers fi +# Add MOCIlib +cp $DRIVER_EXTRACT_DIR/$extract_path/mocilib/*py ./ + # Run our drivers, this produces two command files # driver_envar containing environment variables to be exported # driver_cmd containing a command for the system launcher @@ -109,7 +112,7 @@ for S in $SIGNALS; do trap 'FINALLY' $S done -# Export the enviromnent variables from all the drivers +# Export the environment variables from all the drivers source driver_envar # Run the MPI launcher command and provide timing information if required start=$(date +%s) @@ -117,18 +120,15 @@ source driver_cmd end=$(date +%s) time_in_launcher=$((end-start)) -echo "[DRIVER_TEST_SCRIPT] $time_in_launcher seconds are spent in the" -echo "[DRIVER_TEST_SCRIPT] MPI Launcher" +echo "[DRIVER_TEST_SCRIPT] $time_in_launcher seconds are spent in MPI Launcher" -#export time_in_launcher variable so the cpmip controller can find it -#if required +#export time_in_aprun variable so the cpmip controller can find it if required export time_in_launcher # Run the drivers in finalize mode (after completion of the model). This deals # with model standard out / standard error, and also with any file details # required prior to the start of the next model cycle (if applicable). If -# the MPI launcher has failed, we run the drivers in a stripped down failure -# mode +# MPI launcher has failed, we run the drivers in a stripped down failure mode echo '[DRIVER_TEST_SCRIPT] Finalizing drivers' ./link_drivers --finalize; export DRIVERS_FINALISED=true diff --git a/Coupled_Drivers/unittests/test.py b/Coupled_Drivers/unittests/test.py index ae4d6370..6cc8a164 100755 --- a/Coupled_Drivers/unittests/test.py +++ b/Coupled_Drivers/unittests/test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2022-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -33,13 +33,18 @@ def main(): groups = { 'all': 'test*.py', 'env_lib': 'test_env_lib.py', - 'aprun_command': 'test_aprun_command_construction.py', + 'ocn_lib': 'test_ocn_lib.py', + 'nemo_nl': 'test_update_nemo_nl.py', + 'nemo_driver': 'test_nemo_*.py', + 'nemo_lib': 'test_nemo_*lib*.py', + 'nemo_restart_lib': 'test_nemo_restart_lib*.py', + 'common': 'test_common*.py', + 'dev': 'test_ocn_lib.py', 'mct': 'test_mct_driver.py', 'lfric': 'test_lfric_driver.py', 'xios': 'test_xios_driver.py', 'dependency_checker': 'test_driver_dependencies.py', 'cpmip': 'test_cpmip*py', - 'nemo_driver': 'test_nemo_driver.py', 'um_driver': 'test_um_driver.py', 'cice_driver': 'test_cice_driver.py', 'rivers': 'test_rivers_driver.py', @@ -79,7 +84,7 @@ def main(): test_rtn = unittest.TextTestRunner(buffer=True).run(test_suite) rtn_code += len(test_rtn.failures) + len(test_rtn.errors) - sys.exit(rtn_code) + exit(rtn_code) if __name__ == '__main__': main() diff --git a/Coupled_Drivers/unittests/test_aprun_command_construction.py b/Coupled_Drivers/unittests/test_aprun_command_construction.py index 62402919..12b0e4a7 100644 --- a/Coupled_Drivers/unittests/test_aprun_command_construction.py +++ b/Coupled_Drivers/unittests/test_aprun_command_construction.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -18,12 +18,8 @@ Tests construction of aprun command. ''' import unittest -try: - import unittest.mock as mock -except ImportError: - import mock +import unittest.mock as mock import os -import copy from io import StringIO import common @@ -32,7 +28,6 @@ import nemo_driver import xios_driver import jnr_driver -import lfric_driver #pylint: disable=protected-access @@ -80,7 +75,6 @@ def test_set_aprun_options_single_proc(self): self.assertEqual(opts, expected_output) - class TestUMDriver(unittest.TestCase): ''' Tests for um_driver.py ''' @@ -202,77 +196,6 @@ def test_set_launcher_command_driver_construction_not_aprun(self): self.assertEqual(um_envar["ROSE_LAUNCHER_PREOPTS_UM"], \ expected_output_preopts) - -class TestLFRicDriver(unittest.TestCase): - '''Tests for lfric driver''' - - def setUp(self): - ''' - Setup a common lfric_envar object - ''' - self.lfric_envar = dr_env_lib.env_lib.LoadEnvar() - self.lfric_envar.add('LFRIC_LINK', 'lfric.exe') - self.lfric_envar.add('CONFIG_NL_PATH_LFRIC', 'config_nl_path') - - def test_set_launcher_command_suite_construction(self): - '''Test LFRic driver _set_launcher_command() returns a string with the - correct values if ROSE_LAUNCHER_PREOPTS_LFRIC is set as an environment - variable regardless of the launcher''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'string of preopts') - - expected_output_preopts = "'string of preopts'" - expected_cmd = 'string of preopts ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command(None, - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - - @mock.patch('lfric_driver.common.set_aprun_options') - def test_set_launcher_command_not_aprun(self, mock_set_aprun): - '''Test LFRic driver _set_launcher_command returns no preopts if - ROSE_LAUNCHER_PREOPTS_LFRIC is 'unset', and the launcher is not - aprun''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'unset') - - expected_output_preopts = "''" - expected_cmd = ' ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command('not_aprun', - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - mock_set_aprun.assert_not_called() - - @mock.patch('lfric_driver.common.set_aprun_options') - def test_set_launcher_command_aprun(self, mock_set_aprun): - '''Test LFRic driver _set_launcher_command returns custom preopts if - ROSE_LAUNCHER_PREOPTS_LFRIC is 'unset', and the launcher is - aprun''' - lfric_envar = copy.deepcopy(self.lfric_envar) - lfric_envar.add('ROSE_LAUNCHER_PREOPTS_LFRIC', 'unset') - lfric_envar.add('LFRIC_NPROC', 'lfric_nproc') - lfric_envar.add('LFRIC_NODES', 'lfric_nodes') - lfric_envar.add('OMPTHR_LFRIC', 'lfric_threads') - lfric_envar.add('LFRICHYPERTHREADS', 'lfric_hyper') - - mock_set_aprun.return_value = 'custom_preopts' - - expected_output_preopts = "'custom_preopts'" - expected_cmd = 'custom_preopts ./lfric.exe config_nl_path' - self.assertEqual(lfric_driver._set_launcher_command('aprun', - lfric_envar), - expected_cmd) - self.assertEqual(lfric_envar['ROSE_LAUNCHER_PREOPTS_LFRIC'], - expected_output_preopts) - mock_set_aprun.assert_called_once_with( - 'lfric_nproc', 'lfric_nodes', 'lfric_threads', 'lfric_hyper', False) - - - - class TestNEMODriver(unittest.TestCase): ''' Tests for nemo_driver.py ''' @@ -563,8 +486,8 @@ def test_setup_executable_none(self, _): self._set_xios_nproc() with mock.patch('sys.stdout', new=StringIO()) as mock_out, \ - mock.patch('sys.stderr', new=StringIO()) as _, \ - mock.patch.dict(os.environ, self.xios_envar_dict): + mock.patch('sys.stderr', new=StringIO()) as mock_err, \ + mock.patch.dict(os.environ, self.xios_envar_dict): xios_driver._setup_executable(self.common_env_dict) self.assertRegex(mock_out.getvalue(), \ diff --git a/Coupled_Drivers/unittests/test_common.py b/Coupled_Drivers/unittests/test_common.py index 769ab23c..c7732a6e 100644 --- a/Coupled_Drivers/unittests/test_common.py +++ b/Coupled_Drivers/unittests/test_common.py @@ -110,3 +110,36 @@ def test_remove_latest_hist_dir(self): glob.glob(self.dirpattern), expected_dirs ) + +class TestPathFunctions(unittest.TestCase): + ''' + Test the functions relating to dealing with file paths + ''' + def test_remove_trailing_slash_no_slash(self): + '''Test that with no trailing slash the path string gets passed + straight through''' + path = '/path/with/no/slash' + self.assertEqual(common.remove_trailing_slash(path), path) + + def test_remove_trailing_slash_with_slash(self): + '''Test that we successfully strip the trailing slash''' + path = '/path/with/trailing/slash/' + self.assertEqual(common.remove_trailing_slash(path), path[:-1]) + + def test_remove_three_trailing_slashes(self): + '''Test that we successfully remove three trailing slashes and + no initial forward slash''' + path = 'path/with/three/trailing/slashes///' + self.assertEqual(common.remove_trailing_slash(path), path[:-3]) + + def test_remove_trailing_space(self): + '''Test that we successfully remove trailing whitespacec''' + path = '/path/with/trailing/space ' + self.assertEqual(common.remove_trailing_slash(path), path[:-1]) + + def test_remove_combo(self): + '''Test removal of combination of whitespace and slashes''' + path = '/path/with/trailing/combo/ /\t ///' + expected_return = '/path/with/trailing/combo' + self.assertEqual(common.remove_trailing_slash(path), expected_return) + diff --git a/Coupled_Drivers/unittests/test_envar_lib.py b/Coupled_Drivers/unittests/test_envar_lib.py index 2a23cfa6..91e3702e 100644 --- a/Coupled_Drivers/unittests/test_envar_lib.py +++ b/Coupled_Drivers/unittests/test_envar_lib.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -14,12 +14,7 @@ ''' import unittest -try: - # mock is integrated into unittest as of Python 3.3 - import unittest.mock as mock -except ImportError: - # mock is a standalone package (back-ported) - import mock +import unittest.mock as mock import io import os diff --git a/Coupled_Drivers/unittests/test_link_drivers.py b/Coupled_Drivers/unittests/test_link_drivers.py index a51c9183..88bc3d73 100644 --- a/Coupled_Drivers/unittests/test_link_drivers.py +++ b/Coupled_Drivers/unittests/test_link_drivers.py @@ -62,12 +62,6 @@ class TestChecks(unittest.TestCase): Test the private check methods. ''' - def setUp(self): - pass - - def tearDown(self): - pass - def test_check_drivers_single(self): ''' Assert single model driver is present diff --git a/Coupled_Drivers/unittests/test_nemo_driver.py b/Coupled_Drivers/unittests/test_nemo_driver.py index 88cf1308..c49382a2 100644 --- a/Coupled_Drivers/unittests/test_nemo_driver.py +++ b/Coupled_Drivers/unittests/test_nemo_driver.py @@ -146,7 +146,7 @@ def test_verify_fix_rst_success(self): restart_date = '19931116' expected_msg = '[INFO] Validated NEMO restart date\n' - with mock.patch('sys.stdout', new=io.StringIO()) as output: + with mock.patch('nemo_driver.sys.stdout', new=io.StringIO()) as output: corrected_restart_date = nemo_driver._verify_fix_rst( restart_date, self.nemo_rst, self.model_step_start, self.nemo_step_int, self.nemo_last_step, self.calendar) @@ -174,7 +174,7 @@ def test_verify_fix_rst_fail(self): 'cplhco_19931116_restart_icb_0050.nc' ] - with mock.patch('sys.stdout', new=io.StringIO()) as output: + with mock.patch('nemo_driver.sys.stdout', new=io.StringIO()) as output: corrected_restart_date = nemo_driver._verify_fix_rst( restart_date, self.nemo_rst, self.model_step_start, self.nemo_step_int, self.nemo_last_step, self.calendar) diff --git a/Coupled_Drivers/unittests/test_nemo_driver_misc.py b/Coupled_Drivers/unittests/test_nemo_driver_misc.py new file mode 100644 index 00000000..f4417523 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_misc.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_driver_misc.py + +DESCRIPTION + Test miscellaneous functions in the Nemo Driver +''' + +import unittest +import unittest.mock as mock + +import os + +import error +import nemo_driver + + +class TestNemoDriverAssertions(unittest.TestCase): + ''' + Test the assertions for the NEMO drivers, and that when they fail + an assertion error is raised + ''' + def test_success_nemo_36(self): + '''If the nemo version is 3.6 and NEMO is in parallel mode, we run + correctly''' + nemo_envar = {'NEMO_VERSION': 306, + 'NEMO_NPROC': 10} + self.assertIsNone(nemo_driver._nemo_driver_assertions(nemo_envar)) + + def test_success_nemo_40(self): + '''If the nemo version is 4.0 and NEMO is in parallel mode, we run + correctly''' + nemo_envar = {'NEMO_VERSION': 400, + 'NEMO_NPROC': 2} + self.assertIsNone(nemo_driver._nemo_driver_assertions(nemo_envar)) + + def test_failure_nemo_serial(self): + '''If nemo is attempting to run in serial mode, assertion will fail''' + nemo_envar = {'NEMO_VERSION': 400, + 'NEMO_NPROC': 1} + with self.assertRaises(AssertionError) as context: + nemo_driver._nemo_driver_assertions(nemo_envar) + self.assertEqual('[FAIL] Nemo driver does not support the running' + ' of NEMO in serial mode\n', + str(context.exception)) + + def test_failure_nemo_34(self): + '''If nemo version is lower than 3.6 (ie 3.4) assertion will fail''' + nemo_envar = {'NEMO_VERSION': 304, + 'NEMO_NPROC': 2} + with self.assertRaises(AssertionError) as context: + nemo_driver._nemo_driver_assertions(nemo_envar) + self.assertEqual('[FAIL] The python drivers are only valid for NEMO' + ' versions 3.6 and later.\n', + str(context.exception)) + +class TestRunSubmodelCustom(unittest.TestCase): + ''' + Test the running of code that is custom to the submodels + ''' + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_si3_only( + self, mock_update_si3_nl, mock_update_top_nl): + common_env = {'models': 'si3'} + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', 'ln_restart', 'restart_ctl')) + mock_update_si3_nl.assert_called_once_with('nemo_envar') + mock_update_top_nl.assert_not_called() + + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_top_lnrestart_true( + self, mock_update_si3_nl, mock_update_top_nl): + '''Test the correct running when ln_restart is .true. and no si3''' + common_env = {'models': 'top'} + ln_restart = '.true.' + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', ln_restart, 'restart_ctl')) + mock_update_top_nl.assert_called_once_with( + 'nemo_envar', '.true.', 'restart_ctl', '.false.') + mock_update_si3_nl.assert_not_called() + + @mock.patch('nemo_driver.update_nemo_nl.update_top_nl') + @mock.patch('nemo_driver.update_nemo_nl.update_si3_nl') + def test_run_submodel_custom_top_lnrestart_false_and_si3( + self, mock_update_si3_nl, mock_update_top_nl): + '''Test the correct running when ln_restart is .false. and si3''' + common_env = {'models': 'top si3'} + ln_restart = '.false.' + self.assertIsNone(nemo_driver._run_submodel_custom( + common_env, 'nemo_envar', ln_restart, 'restart_ctl')) + mock_update_top_nl.assert_called_once_with( + 'nemo_envar', '.false.', 'restart_ctl', '.true.') + mock_update_si3_nl.assert_called_once_with('nemo_envar') + + @mock.patch('nemo_driver.sys.stderr.write') + def test_run_submodel_custom_top_lnrestart_invalid(self, mock_stderr): + '''Test that we exit with an error message if ln_restart has invalid + value''' + common_env = {'models': 'top'} + ln_restart = 'invalid' + with self.assertRaises(SystemExit) as context: + nemo_driver._run_submodel_custom(common_env, None, ln_restart, None) + self.assertEqual(context.exception.code, + error.INVALID_LOCAL_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] TOP: invalid ln_restart value: invalid\n') + + + +class TestProcessorRestartFiles(unittest.TestCase): + ''' + Test the call to set up the various restart files + ''' + @mock.patch('nemo_driver.ocn_lib.setup_restart') + def test_processor_restart_files_crun(self, mock_setup_rst): + '''Check the correct calls to setup_restart for CRUN''' + nemo_driver._processor_restart_files_crun( + 'nemo_init_dir', 'rst_link_dir', 'nemo_nproc', + 'nemo_dump_time', 'runid') + mock_setup_rst.assert_has_calls( + [mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_nemo_dump_time_restart', + 'restart', 'nemo_nproc', 'NEMO'), + mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_nemo_dump_time_restart_ice', + 'restart_ice', 'nemo_nproc', 'SI3'), + mock.call('nemo_init_dir', 'rst_link_dir', + 'runido_icebergs_nemo_dump_time_restart', + 'restart_icebergs', 'nemo_nproc', 'Icebergs')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_nemo_only(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with no Ice or + TOP''' + mock_setup_nrun.side_effect = ['ln_restart', None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', + 'restart_icebergs', 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_with_si3(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with Ice but no + TOP''' + mock_setup_nrun.side_effect = ['ln_restart', None, None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'SI3_START': 'si3_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios si3'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', 'restart_icebergs', + 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n'), + mock.call( + 'si3_startdump', 'rst_link_dir', 'restart_ice', + 'nemo_init_dir', 'New SI3 run\n')]) + + @mock.patch('nemo_driver.ocn_lib.setup_nrun') + def test_processor_restart_files_nrun_with_top(self, mock_setup_nrun): + '''Check the correct calls to setup_restart for NRUN with TOP but no + ice''' + mock_setup_nrun.side_effect = ['ln_restart', None, None] + nemo_envar = {'NEMO_START': 'nemo_startdump', + 'NEMO_ICEBERGS_START': 'icebergs_startdump', + 'TOP_START': 'top_startdump', + 'RST_LINK_DIR': 'rst_link_dir'} + common_env = {'models': 'nemo xios top'} + self.assertEqual('ln_restart', + nemo_driver._processor_restart_files_nrun( + nemo_envar, common_env, 'nemo_init_dir')) + mock_setup_nrun.assert_has_calls( + [mock.call('nemo_startdump', 'rst_link_dir', + 'restart', 'nemo_init_dir', + 'NEMO_START not set\nNEMO will use climatology\n'), + mock.call( + 'icebergs_startdump', 'rst_link_dir', + 'restart_icebergs', 'nemo_init_dir', + 'NEMO_ICEBERGS_START not set or file(s)' \ + ' not found. Icebergs (if switched on) will start' \ + ' from a state of zero icebergs\n'), + mock.call( + 'top_startdump', 'rst_link_dir', 'restart_trc', + 'nemo_init_dir', 'New TOP run\n')]) + + +class TestCrunTimesteps(unittest.TestCase): + ''' + Test the setting up of timesteps for a CRUN + ''' + @mock.patch('nemo_driver.sys.stdout.write') + def test_integer_cycle_msg_continue_from_fail(self, mock_stdout): + '''Test an integer cycling when used with CONTINUE_FROM_FAIL true''' + common_env = {'CONTINUE_FROM_FAIL': 'true'} + expected_out = ('.true.', 2, 145) + rvalue = nemo_driver._crun_timesteps( + False, '144', 1200, None, common_env) + self.assertEqual(rvalue, expected_out) + mock_stdout.assert_called_once_with( + '[INFO] Nemo has previously completed 2 days\n') + + @mock.patch('nemo_driver.sys.stdout.write') + def test_date_cycle(self, mock_stdout): + '''Test an date cycling when used with CONTINUE_FROM_FAIL false''' + common_env = {'CONTINUE_FROM_FAIL': 'false'} + expected_out = ('.true.', 2, 73) + rvalue = nemo_driver._crun_timesteps( + True, None, None, 72, common_env) + self.assertEqual(rvalue, expected_out) + mock_stdout.assert_not_called() + +class TestNrunTimesteps(unittest.TestCase): + ''' + Test setting up of timesteps for an NRUN + ''' + def test_nrun_timesteps(self): + nemo_first_step = 72 + expected_return = (0, nemo_first_step, nemo_first_step - 1) + self.assertEqual( + expected_return, nemo_driver._nrun_timesteps(nemo_first_step)) + + +class TestWriteOutputFileToStdOut(unittest.TestCase): + ''' + Test the writing of an ocean output file to standard out + ''' + def setUp(self): + '''Create an output, one with unicode, one with errors to read''' + self.unicode_fn = 'unicode_output' + self.error_fn = 'error_output' + self.unicode_contents = '''A first line in the ascii range +A second line in the ascii range +A third line containing a unicode division in brackets (\u00F7) +A fourth line in the ascii range''' + self.error_contents = '''A first line with no error +A second line with E R R O R +A a third line with no error +A fourth line with E R R O R in the middle +A fifth line''' + with open(self.unicode_fn, 'w', encoding='utf8') as unicode_fh: + unicode_fh.write(self.unicode_contents) + with open(self.error_fn, 'w') as error_fh: + error_fh.write(self.error_contents) + + def tearDown(self): + '''Remove the test files''' + for test_f in [self.unicode_fn, self.error_fn]: + try: + os.remove(test_f) + except FileNotFoundError: + pass + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_error_zero(self, mock_stdout): + '''Test errors get written and we count two of them with no + error count provided''' + self.assertEqual( + 2, nemo_driver._write_output_file_to_stdout(self.error_fn)) + mock_stdout.assert_has_calls( + [mock.call( + '[INFO] Ocean output from file %s\n' % self.error_fn), + mock.call('A first line with no error\n'), + mock.call('A second line with E R R O R\n'), + mock.call('A a third line with no error\n'), + mock.call('A fourth line with E R R O R in the middle\n'), + mock.call('A fifth line')]) + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_error_non_zero(self, mock_stdout): + '''Test errors get written and we count four of them when input + error count is 2''' + self.assertEqual( + 4, nemo_driver._write_output_file_to_stdout(self.error_fn, 2)) + + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_output_file_to_stdout_unicode_error_passed_in( + self, mock_stdout): + '''Test that the unicode string is written successfully without the + unicode character, and that with 2 errors passed in, 2 are passed out + as there are no E R R O Rs in this test''' + self.assertEqual( + 2, nemo_driver._write_output_file_to_stdout(self.unicode_fn, 2)) + mock_stdout.assert_has_calls( + [mock.call( + '[INFO] Ocean output from file %s\n' % self.unicode_fn), + mock.call('A first line in the ascii range\n'), + mock.call('A second line in the ascii range\n'), + mock.call('A third line containing a unicode division in' + ' brackets ()\n'), + mock.call('A fourth line in the ascii range')]) + +class TestWriteOceanToStdout(unittest.TestCase): + ''' + Test the listing through of output files and calling the function to write + them to standard out + ''' + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_succeed( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test for all potential restart files exisiting''' + mock_isfile.side_effect = [True, True, True, True] + mock_write_file.side_effect = ['error1', 'error2', 'error3', 'error4'] + self.assertEqual('error4', nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_has_calls( + [mock.call('ocean.output', 0), + mock.call('solver.stat', 'error1'), + mock.call('run.stat', 'error2'), + mock.call('icebergs.stat', 'error3')]) + mock_stdout.assert_not_called() + + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_failed( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test for all potential restart files missing''' + no_file_line = '[INFO] Nemo output file %s not avaliable\n' + mock_isfile.side_effect = [False, False, False, False] + self.assertEqual(0, nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_not_called() + mock_stdout.assert_has_calls( + [mock.call(no_file_line % 'ocean.output'), + mock.call(no_file_line % 'solver.stat'), + mock.call(no_file_line % 'run.stat'), + mock.call(no_file_line % 'icebergs.stat')]) + + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._write_output_file_to_stdout') + @mock.patch('nemo_driver.sys.stdout.write') + def test_write_ocean_out_to_stdout_nemo4_combo( + self, mock_stdout, mock_write_file, mock_isfile): + '''Test the file combination for NEMO4 (no solver.stat)''' + no_file_line = '[INFO] Nemo output file %s not avaliable\n' + mock_isfile.side_effect = [True, False, True, True] + mock_write_file.side_effect = ['error1', 'error2', 'error3'] + self.assertEqual('error3', nemo_driver._write_ocean_out_to_stdout()) + mock_isfile.assert_has_calls([mock.call('ocean.output'), + mock.call('solver.stat'), + mock.call('run.stat'), + mock.call('icebergs.stat')]) + mock_write_file.assert_has_calls( + [mock.call('ocean.output', 0), + mock.call('run.stat', 'error1'), + mock.call('icebergs.stat', 'error2')]) + mock_stdout.assert_called_once_with(no_file_line % 'solver.stat') + + +class TestMoveNLEndOfRun(unittest.TestCase): + ''' + Test the copying of the namelist files at the end of the run + ''' + def setUp(self): + '''Set up the environment variable object''' + self.nemo_envar_fin = {'NEMO_NL': 'nemo_namelist', + 'TOP_NL': 'top_namelist'} + + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_nemo_only(self, mock_copy, mock_isdir): + '''Test copying of namelist NEMO only''' + mock_isdir.return_value = True + nemo_driver._copy_nl_end_of_run(self.nemo_envar_fin, 'nemo_rst', None) + mock_isdir.assert_called_once_with('nemo_rst') + mock_copy.assert_called_once_with('nemo_namelist', 'nemo_rst') + + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_nemo_top(self, mock_copy, mock_isdir): + '''Test copying of namelist NEMO and TOP''' + mock_isdir.side_effect = [True, True] + nemo_driver._copy_nl_end_of_run( + self.nemo_envar_fin, 'nemo_rst', 'top_rst') + mock_isdir.assert_has_calls([mock.call('nemo_rst'), + mock.call('top_rst')]) + mock_copy.assert_has_calls([mock.call('nemo_namelist', 'nemo_rst'), + mock.call('top_namelist', 'top_rst')]) + + + + +class TestFinaliseExecutable(unittest.TestCase): + ''' + Test the finalise executable function + ''' + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.sys.stderr.write') + def test_finalize_executable_error_count( + self, mock_stderr, mock_write, mock_cwd, mock_stdout): + '''Test we exit if the error count is one''' + mock_write.return_value = 1 + mock_cwd.return_value = 'current_wd' + with self.assertRaises(SystemExit) as context: + nemo_driver._finalize_executable(None) + mock_stdout.assert_has_calls( + [mock.call('[INFO] finalizing NEMO\n'), + mock.call('[INFO] running finalize in current_wd\n')]) + mock_cwd.assert_called_once_with() + self.assertEqual(context.exception.code, + error.COMPONENT_MODEL_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] An error has been found with the NEMO run.' + ' Please investigate the ocean.output file for more' + ' details\n') + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver._copy_nl_end_of_run') + def test_finalize_executable_nemo_rst_top_rist( + self, mock_copy_nl_end_of_run, mock_read_nl, + mock_load_env, mock_write, mock_cwd, mock_stdout): + '''Test finalise when the nemo and top restarts are avaliable to move''' + common_env = {'models': 'model list'} + mock_write.return_value = 0 + mock_cwd.return_value = '' + mock_load_env.return_value = ({'NEMO_NL': 'nemo_namelist'}, + 'model list updated') + mock_read_nl.return_value = ('nemo_rst', None, None, 'top_rst') + nemo_driver._finalize_executable(common_env) + mock_stdout.assert_has_calls( + [mock.call('[INFO] finalizing NEMO\n'), + mock.call('[INFO] running finalize in \n')]) + mock_load_env.assert_called_once_with('final', 'model list') + mock_read_nl.assert_called_once_with(common_env, + {'NEMO_NL': 'nemo_namelist'}) + mock_copy_nl_end_of_run.assert_called_once_with( + {'NEMO_NL': 'nemo_namelist'}, 'nemo_rst', 'top_rst') + + @mock.patch('nemo_driver.sys.stdout.write') + @mock.patch('nemo_driver.os.getcwd') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.os.path.isdir') + @mock.patch('nemo_driver.shutil.copy') + def test_finalize_executable_nemo_rst_no_nemo_rst( + self, mock_copy, mock_isdir, mock_read_nl, + mock_load_env, mock_write, mock_cwd, mock_stdout): + '''Test finalise when there is no nemo_rst directory''' + mock_write.return_value = 0 + mock_cwd.return_value = '' + mock_load_env.return_value = ({'NEMO_NL': 'nemo_namelist'}, + 'model list updated') + mock_read_nl.return_value = ('nemo_rst', None, None, None) + mock_isdir.return_value = False + nemo_driver._finalize_executable({'models': 'model list updated'}) + mock_copy.assert_not_called() + +class TestRunDriver(unittest.TestCase): + ''' + Test the interface to run the driver + ''' + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_finalize(self, mock_write, mock_finalize, mock_setup): + '''Test finalise mode''' + rvalue = nemo_driver.run_driver('common_env', 'finalize', 'run_info') + self.assertEqual(rvalue, (None, None, 'run_info', None)) + mock_setup.assert_not_called() + mock_write.assert_not_called() + mock_finalize.assert_called_once_with('common_env') + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_failure(self, mock_write, mock_finalize, mock_setup): + '''Test finalise mode''' + rvalue = nemo_driver.run_driver('common_env', 'failure', 'run_info') + self.assertEqual(rvalue, (None, None, 'run_info', None)) + mock_setup.assert_not_called() + mock_finalize.assert_not_called() + mock_write.assert_called_once_with() + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._set_launcher_command') + @mock.patch('nemo_driver.nemo_runtime_namcouple.sent_coupling_fields') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_l_namcouple(self, mock_write, mock_finalize, mock_namc, + mock_launcher_cmd, mock_setup): + '''Test run mode with l_namcouple set in run info''' + run_info = {'l_namcouple': True} + common_env = {'ROSE_LAUNCHER': 'launcher'} + mock_setup.return_value = 'exe_envar' + mock_launcher_cmd.return_value = 'launch_cmd' + rvalue = nemo_driver.run_driver(common_env, 'run_driver', run_info) + self.assertEqual(rvalue, ('exe_envar', 'launch_cmd', run_info, None)) + mock_setup.assert_called_once_with(common_env) + mock_launcher_cmd.assert_called_once_with('launcher', 'exe_envar') + mock_namc.assert_not_called() + mock_finalize.assert_not_called() + mock_write.assert_not_called() + + @mock.patch('nemo_driver._setup_executable') + @mock.patch('nemo_driver._set_launcher_command') + @mock.patch('nemo_driver.nemo_runtime_namcouple.sent_coupling_fields') + @mock.patch('nemo_driver._finalize_executable') + @mock.patch('nemo_driver._write_ocean_out_to_stdout') + def test_run_driver_no_l_namcouple( + self, mock_write, mock_finalize, mock_namc, mock_launcher_cmd, + mock_setup): + '''Test run mode with l_namcouple set to false in run info''' + run_info = {'l_namcouple': False} + common_env = {'ROSE_LAUNCHER': 'launcher'} + mock_setup.return_value = 'exe_envar' + mock_launcher_cmd.return_value = 'launch_cmd' + mock_namc.return_value = ('run_info', 'model_snd_list') + rvalue = nemo_driver.run_driver(common_env, 'run_driver', run_info) + self.assertEqual( + rvalue, ('exe_envar', 'launch_cmd', 'run_info', 'model_snd_list')) + mock_setup.assert_called_once_with(common_env) + mock_launcher_cmd.assert_called_once_with('launcher', 'exe_envar') + mock_namc.assert_called_once_with('exe_envar', run_info) + mock_finalize.assert_not_called() + mock_write.assert_not_called() + +class TestVerifyRestart(unittest.TestCase): + ''' + Test the verification of NEMO restart files + ''' + @mock.patch('nemo_driver.nemo_restart_lib.verify_fix_rst') + def test_verify_restart_no_validation(self, mock_verify_fix): + '''Test for DRIVERS_VERIFY_RST false, nothing is done''' + common_env = {'DRIVERS_VERIFY_RST': 'False'} + self.assertEqual( + 'nemo_dump_time', nemo_driver._verify_restart( + common_env, None, 'nemo_dump_time', None)) + mock_verify_fix.assert_not_called() + + @mock.patch('nemo_driver.nemo_restart_lib.verify_fix_rst') + def test_verify_restart(self, mock_verify_fix): + '''Test for DRIVERS_VERIFY_RST True, validation performed''' + common_env = {'DRIVERS_VERIFY_RST': 'True', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point'} + nemo_env = {'NEMO_NPROC': '25'} + mock_verify_fix.return_value = 'nemo_dump_time_rtn' + self.assertEqual( + 'nemo_dump_time_rtn', nemo_driver._verify_restart( + common_env, nemo_env, 'nemo_dump_time', 'nemo_rst')) + mock_verify_fix.assert_called_once_with( + 'nemo_dump_time', 'cycle_point', 'nemo_rst') + +class TestSetLauncherCommand(unittest.TestCase): + ''' + Test the setting up of the launcher command for NEMO executable + ''' + @mock.patch('nemo_driver.common.set_aprun_options') + def test_set_launcher_command_aprun_unset(self, mock_set_aprun): + '''Test with aprun launcher and ROSE_LAUNCHER_PREOPTS_NEMO unset''' + mock_set_aprun.return_value = 'aprun -n 240 -N 24 -S 12 -d 1' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'unset', + 'NEMO_NPROC': '240', + 'OCEAN_NODES': '10', + 'OMPTHR_OCN': '1', + 'OHYPERTHREADS': 'false', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'aprun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command + self.assertEqual(result, 'aprun -n 240 -N 24 -S 12 -d 1 ./nemo.exe') + # Check that nemo_envar was updated and quoted + self.assertEqual( + nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], + "'aprun -n 240 -N 24 -S 12 -d 1'") + # Check that set_aprun_options was called correctly + mock_set_aprun.assert_called_once_with('240', '10', '1', 'false', False) + + def test_set_launcher_command_non_aprun_unset(self): + '''Test with non-aprun launcher and ROSE_LAUNCHER_PREOPTS_NEMO unset''' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'unset', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'srun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command (empty preopts) + self.assertEqual(result, ' ./nemo.exe') + # Check that nemo_envar was updated and quoted + self.assertEqual(nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], "''") + + def test_set_launcher_command_preopts_already_set(self): + '''Test when ROSE_LAUNCHER_PREOPTS_NEMO is already configured''' + nemo_envar = { + 'ROSE_LAUNCHER_PREOPTS_NEMO': 'mpirun -np 100', + 'OCEAN_LINK': 'nemo.exe' + } + launcher = 'aprun' + result = nemo_driver._set_launcher_command(launcher, nemo_envar) + + # Check the returned launch command uses existing preopts + self.assertEqual(result, 'mpirun -np 100 ./nemo.exe') + # Check that nemo_envar was quoted + self.assertEqual( + nemo_envar['ROSE_LAUNCHER_PREOPTS_NEMO'], "'mpirun -np 100'") diff --git a/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py b/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py new file mode 100644 index 00000000..fc099c73 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_setup_exe.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_driver_setup_exe.py + +DESCRIPTION + Test the setup executable function in NEMO driver. This has been separated + out, the function is essentially a list of calls to other functions but + it requires a large degree of mocking. +''' + +import unittest +import unittest.mock as mock + +import nemo_driver + + +class TestSetupExecutable(unittest.TestCase): + ''' + Test the setup_executable function + ''' + def setUp(self): + '''Commonalities required for setup executable''' + self.nemo_envar = {'OCEAN_LINK': 'ocean_link', + 'OCEAN_EXEC': 'ocean_exec', + 'NEMO_NL': 'nemo_namelist', + 'NEMO_NPROC': '25', + 'RST_LINK_DIR': 'rst_link_dir'} + self.nemo_envar_si3 = {**self.nemo_envar, + **{'SI3_NL': 'ice_namelist'}} + self.common_env = {'RUNID': 'runid', + 'models': 'nemo xios'} + self.common_env_si3 = {'RUNID': 'runid', + 'models': 'nemo si3 xios'} + + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.common.remove_file') + @mock.patch('nemo_driver.os.symlink') + @mock.patch('nemo_driver.nemo_lib.setup_dates') + @mock.patch('nemo_driver._nemo_driver_assertions') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.nemo_restart_lib.create_restart_direcs') + @mock.patch('nemo_driver.nemo_restart_lib.compile_nemo_restart_files') + @mock.patch('nemo_driver.nemo_restart_lib.setup_previous_restart_nl') + @mock.patch('nemo_driver.os.path.join') + @mock.patch('nemo_driver.nemo_lib.read_history_nl') + # Beginning of IF os.path.isfile + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver._processor_restart_files_nrun') + @mock.patch('nemo_driver._nrun_timesteps') + # End if + @mock.patch('nemo_driver.nemo_lib.setup_nemo_runlen') + @mock.patch('nemo_driver.update_nemo_nl.update_nl') + # Not called + @mock.patch('nemo_driver._verify_restart') + @mock.patch('nemo_driver._processor_restart_files_crun') + @mock.patch('nemo_driver._crun_timesteps') + def test_setup_executable_nrun_no_si3(self, + mock_crun_timesteps, + mock_proc_files_crun, + mock_verify_rst, + # Not called + mock_update_nl, + mock_setup_runlen, + # End if + mock_nrun_timesteps, + mock_proc_files_nrun, + mock_path_isfile, + # Beginning of IF os.path.isfile + mock_read_history_nl, + mock_path_join, + mock_setup_previous_nl, + mock_compile_restarts, + mock_create_restart_direcs, + mock_read_current_nl, + mock_assertions, + mock_setup_dates, + mock_symlink, + mock_remove_file, + mock_load_envar): + '''Test setting up of executable for an NRUN with no SI3''' + mock_load_envar.return_value = (self.nemo_envar, \ + self.common_env['models']) + # The list is model basis + mock_setup_dates.return_value = ( + 'nleapy', [2020, 12, 1, 0, 0, 0], 'run_start', 'run_length', + 'run_days') + mock_read_current_nl.return_value = ('nemo_rst', None, None, None) + mock_compile_restarts.return_value = ( + 'nemo_restart_files', 'latest_nemo_dump', 'nemo_init_dir') + mock_setup_previous_nl.return_value = 'restart_nl_path' + mock_path_join.return_value = 'history_nemo_nl' + mock_read_history_nl.return_value = ( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date_bool') + # Beginning of IF os.path.isfile + mock_path_isfile.return_value = False + mock_proc_files_nrun.return_value = 'ln_restart' + mock_nrun_timesteps.return_value = ( + 'restart_ctl', 'nemo_next_step', 'nemo_last_step') + # End if + mock_setup_runlen.return_value = 'nemo_final_step' + + self.assertEqual( + self.nemo_envar, nemo_driver._setup_executable(self.common_env)) + + mock_load_envar.assert_called_once_with('setup', 'nemo xios') + mock_remove_file.assert_called_once_with('ocean_link') + mock_symlink.assert_called_once_with('ocean_exec', 'ocean_link') + mock_read_current_nl.assert_called_once_with(self.common_env, + self.nemo_envar) + mock_create_restart_direcs.assert_called_once_with( + ['nemo_rst'], self.common_env) + mock_compile_restarts.assert_called_once_with('nemo_rst') + mock_setup_previous_nl.assert_called_once_with( + self.common_env, 'nemo_rst', None, None, 'latest_nemo_dump') + mock_path_join.assert_called_once_with('restart_nl_path', + 'nemo_namelist') + mock_read_history_nl.assert_called_once_with('history_nemo_nl') + # Beginning of IF os.path.isfile + mock_path_isfile.assert_called_once_with('latest_nemo_dump') + mock_proc_files_nrun.assert_called_once_with(self.nemo_envar, + self.common_env, + 'nemo_init_dir') + mock_nrun_timesteps.assert_called_once_with('nemo_first_step') + # End if + mock_setup_runlen.assert_called_once_with( + self.common_env, 'run_start', [2020, 12, 1, 0, 0, 0], + 'nemo_step_int', 'run_days', 'run_length', 'nemo_next_step', + 'nemo_last_step') + mock_update_nl.assert_called_once_with( + self.common_env, self.nemo_envar, 'ln_restart', 'restart_ctl', + 'nemo_next_step', 'nemo_final_step', '20201201', 'nleapy') + + # Not called + mock_verify_rst.assert_not_called() + mock_proc_files_crun.assert_not_called() + mock_crun_timesteps.assert_not_called() + + + @mock.patch('nemo_driver.nemo_lib.load_environment_variables') + @mock.patch('nemo_driver.common.remove_file') + @mock.patch('nemo_driver.os.symlink') + @mock.patch('nemo_driver.nemo_lib.setup_dates') + @mock.patch('nemo_driver._nemo_driver_assertions') + @mock.patch('nemo_driver.nemo_lib.read_current_cycle_nl') + @mock.patch('nemo_driver.nemo_restart_lib.create_restart_direcs') + @mock.patch('nemo_driver.nemo_restart_lib.compile_nemo_restart_files') + @mock.patch('nemo_driver.nemo_restart_lib.setup_previous_restart_nl') + @mock.patch('nemo_driver.os.path.join') + @mock.patch('nemo_driver.nemo_lib.read_history_nl') + # Beginning of IF os.path.isfile + @mock.patch('nemo_driver.os.path.isfile') + @mock.patch('nemo_driver.re.findall') + @mock.patch('nemo_driver._verify_restart') + @mock.patch('nemo_driver._processor_restart_files_crun') + @mock.patch('nemo_driver._crun_timesteps') + # End if + @mock.patch('nemo_driver.nemo_lib.setup_nemo_runlen') + @mock.patch('nemo_driver.update_nemo_nl.update_nl') + # Not called + @mock.patch('nemo_driver._processor_restart_files_nrun') + @mock.patch('nemo_driver._nrun_timesteps') + @mock.patch('nemo_driver._run_submodel_custom') + def test_setup_executable_crun_si3(self, + mock_run_submodel_cust, + mock_nrun_timesteps, + mock_proc_files_nrun, + # Not called + mock_update_nl, + mock_setup_runlen, + # End if + mock_crun_timesteps, + mock_proc_files_crun, + mock_verify_rst, + mock_re_findall, + mock_path_isfile, + # Beginning of IF os.path.isfile + mock_read_history_nl, + mock_path_join, + mock_setup_previous_nl, + mock_compile_restarts, + mock_create_restart_direcs, + mock_read_current_nl, + mock_assertions, + mock_setup_dates, + mock_symlink, + mock_remove_file, + mock_load_envar): + '''Test setting up of executable for a CRUN with si3''' + mock_load_envar.return_value = (self.nemo_envar_si3, + self.common_env_si3['models']) + # The list is model basis + mock_setup_dates.return_value = ( + 'nleapy', [2020, 12, 1, 0, 0, 0], 'run_start', 'run_length', + 'run_days') + mock_read_current_nl.return_value = ( + 'nemo_rst', None, 'ice_rst', None) + mock_compile_restarts.return_value = ( + 'nemo_restart_files', 'latest_nemo_dump', 'nemo_init_dir') + mock_setup_previous_nl.return_value = 'restart_nl_path' + mock_path_join.return_value = 'history_nemo_nl' + mock_read_history_nl.return_value = ( + 'nemo_first_step', 'nemo_last_step', 'nemo_step_int', + 'nemo_rst_date_bool') + # Beginning of IF os.path.isfile + mock_path_isfile.return_value = True + mock_re_findall.return_value = ['nemo_dump_time_re'] + mock_verify_rst.return_value = 'nemo_dump_time_verify' + mock_crun_timesteps.return_value = ( + 'ln_restart', 'restart_ctl', 'nemo_next_step') + # End if + mock_setup_runlen.return_value = 'nemo_final_step' + + self.assertEqual( + self.nemo_envar_si3, + nemo_driver._setup_executable(self.common_env_si3)) + + mock_load_envar.assert_called_once_with('setup', 'nemo si3 xios') + mock_remove_file.assert_has_calls([mock.call('ocean_link'), + mock.call('restart.nc'), + mock.call('restart_ice.nc')]) + mock_symlink.assert_called_once_with('ocean_exec', 'ocean_link') + mock_read_current_nl.assert_called_once_with(self.common_env_si3, + self.nemo_envar_si3) + mock_create_restart_direcs.assert_called_once_with( + ['nemo_rst', 'ice_rst'], self.common_env_si3) + mock_compile_restarts.assert_called_once_with('nemo_rst') + mock_setup_previous_nl.assert_called_once_with( + self.common_env_si3, 'nemo_rst', 'ice_rst', + None, 'latest_nemo_dump') + mock_path_join.assert_called_once_with('restart_nl_path', + 'nemo_namelist') + mock_read_history_nl.assert_called_once_with('history_nemo_nl') + # Beginning of IF os.path.isfile + mock_path_isfile.assert_called_once_with('latest_nemo_dump') + mock_re_findall.assert_called_once_with( + r'_(\d*)_restart', 'latest_nemo_dump') + mock_verify_rst.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'nemo_dump_time_re', + 'nemo_rst') + mock_proc_files_crun.assert_called_once_with( + 'nemo_init_dir', 'rst_link_dir', 25, 'nemo_dump_time_verify', + 'runid') + mock_crun_timesteps.assert_called_once_with( + 'nemo_rst_date_bool', 'nemo_dump_time_verify', 'nemo_step_int', + 'nemo_last_step', self.common_env_si3) + # End if + mock_setup_runlen.assert_called_once_with( + self.common_env_si3, 'run_start', [2020, 12, 1, 0, 0, 0], + 'nemo_step_int', 'run_days', 'run_length', 'nemo_next_step', + 'nemo_last_step') + mock_update_nl.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'ln_restart', + 'restart_ctl', 'nemo_next_step', 'nemo_final_step', '20201201', + 'nleapy') + mock_run_submodel_cust.assert_called_once_with( + self.common_env_si3, self.nemo_envar_si3, 'ln_restart', + 'restart_ctl') + + # Not called + mock_proc_files_nrun.assert_not_called() + mock_nrun_timesteps.assert_not_called() diff --git a/Coupled_Drivers/unittests/test_nemo_driver_submod.py b/Coupled_Drivers/unittests/test_nemo_driver_submod.py new file mode 100644 index 00000000..b82f71e3 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_driver_submod.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' +import unittest +import unittest.mock as mock + +import nemo_driver + +class TestRunSubmodelControllers(unittest.TestCase): + ''' + Test the code that runs the sub model controllers from the nemo drivers + ''' + def setUp(self): + '''Set up a mock controller object to test the run controller''' + class MockController: + '''A mock controller to test the calls to run controller''' + def __init__(self, submodel): + '''Inititalise with a string containing the submodel name''' + self.name = submodel + self.expected_arguments = () + def run_controller(self, *arguments): + '''A function to test the running of the controller''' + assert arguments == self.expected_arguments + def clear(self): + '''Clear expected_arguments''' + self.expected_arguments = () + self.top_controller = MockController('TOP') + self.si3_controller = MockController('SI3') + + @mock.patch('nemo_driver.importlib.import_module') + @mock.patch('nemo_driver.sys.stdout') + def test_no_controllers(self, mock_stdout, mock_import): + nemo_envar = {'L_OCN_PASS_TRC': 'False'} + common_envar = {'models': 'um nemo'} + + nemo_driver._run_submodel_controllers( + nemo_envar, common_envar, None, None, None + ) + + mock_stdout.write.assert_called_once_with( + '[INFO] nemo_driver: Passive tracer code not active.\n' + ) + mock_import.assert_not_called() + + + @mock.patch('nemo_driver.importlib.import_module') + @mock.patch('nemo_driver.sys.stdout') + def test_top_only(self, mock_stdout, mock_import): + '''Test that a top only call works, with upper case T''' + nemo_envar = {'L_OCN_PASS_TRC': 'True', + 'NEMO_NPROC': '10'} + common_envar = {'models': 'um nemo', + 'RUNID': 'runid', + 'DRIVERS_VERIFY_RST': 'verify_rst'} + #expected_arguments_to_controller + expected_args = (common_envar, 'restart_ctl', 10, 'runid', + 'verify_rst', 'nemo_dump_time', 'mode') + self.top_controller.expected_arguments = expected_args + mock_import.return_value = self.top_controller + + nemo_driver._run_submodel_controllers(nemo_envar, common_envar, + 'restart_ctl', 'nemo_dump_time', + 'mode') + mock_stdout.write.assert_has_calls( + [mock.call('[INFO] nemo_driver: Passive tracer code is active.\n'), + mock.call('[INFO] Calling top_controller in mode mode\n')]) + mock_import.assert_called_once_with('top_controller') + self.top_controller.clear() diff --git a/Coupled_Drivers/unittests/test_nemo_lib.py b/Coupled_Drivers/unittests/test_nemo_lib.py new file mode 100644 index 00000000..5987c6d2 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_lib.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_lib.py + +DESCRIPTION + Test the non 'namelist' functions in the NEMO library +''' + +import unittest +import unittest.mock as mock + +import dr_env_lib.nemo_def +import nemo_lib +import error + + +class TestSetupDates(unittest.TestCase): + ''' + Test the setting up of dates for NEMO model run + ''' + def setUp(self): + ''' + Set up common_env dictionary containing model basis, run start, + run length. Calendar type to be added later + ''' + self.common_env = {'MODELBASIS': '2020,12,12,0,0,0', + 'TASKSTART': '2021,01,02,0,0,0', + 'TASKLENGTH': '3,4,5,0,0,0'} + # return values + self.basisls = [2020, 12, 12, 0, 0, 0] + self.startls = [2021, 1, 2, 0, 0, 0] + self.lenls = [3, 4, 5, 0, 0, 0] + + @mock.patch('nemo_lib.sys.stderr.write') + def test_setup_dates_invalid_calendar(self, mock_stderr): + '''Test failure if calendar is invalid''' + common_env = {'CALENDAR': 'invalid_calendar'} + with self.assertRaises(SystemExit) as context: + nemo_lib.setup_dates(common_env) + self.assertEqual(context.exception.code, error.INVALID_EVAR_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Calendar type %s not recognised\n' % common_env['CALENDAR']) + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_gregorian(self, mock_inc_days): + '''Test correct behaviour for gregorian calendar''' + common_env = {**{'CALENDAR': 'gregorian'}, **self.common_env} + expected_return = (1, self.basisls, self.startls, self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, 'gregorian') + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_360d(self, mock_inc_days): + '''Test correct behaviour for 360 day calendar''' + common_env = {**{'CALENDAR': '360day'}, **self.common_env} + expected_return = (30, self.basisls, self.startls, + self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, '360') + + @mock.patch('nemo_lib.inc_days.inc_days', return_value='rundays') + def test_setup_dates_365d(self, mock_inc_days): + '''Test correct behaviour for 365 day calendar''' + common_env = {**{'CALENDAR': '365day'}, **self.common_env} + expected_return = (0, self.basisls, self.startls, + self.lenls, 'rundays') + self.assertEqual(expected_return, nemo_lib.setup_dates(common_env)) + mock_inc_days.assert_called_once_with(2021, 1, 2, 3, 4, 5, '365') + + + + +class TestSetupNemoRunlen(unittest.TestCase): + ''' + Test the setting up of NEMO runlength for climate and coupled nwp style + cycling + ''' + def test_setup_nemo_runlen_climate_cycling(self): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is false''' + common_env = {'CONTINUE_FROM_FAIL': 'false'} + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_step_int = 1200. + nemo_last_step = 2232 + self.assertEqual(2307, + nemo_lib.setup_nemo_runlen( + common_env, None, None, nemo_step_int, run_days, + run_length, None, nemo_last_step)) + + def test_setup_nemo_runlen_cnwp_cycling(self): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is true + and the warning is not called''' + common_env = {'CONTINUE_FROM_FAIL': 'true', + 'LAST_DUMP_HOURS': '2'} + run_start = [2020, 1, 1, 1, 0, 0] + model_basis = [2019, 11, 1, 0, 0, 0] + nemo_step_int = 1200. + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_next_step = 4402 + self.assertEqual(4470, + nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, None)) + + @mock.patch('nemo_lib.sys.stderr.write') + def test_setup_nemo_runlen_cnwp_cycling_failure(self, mock_stderr): + '''Test the correct behaviour when CONTINUE_FROM_FAIL is true and the + warning is called''' + common_env = {'CONTINUE_FROM_FAIL': 'true', + 'LAST_DUMP_HOURS': '2'} + run_start = [2020, 1, 1, 1, 0, 0] + model_basis = [2019, 11, 1, 0, 0, 0] + nemo_step_int = 1200. + run_days = 1 + run_length = [None, None, None, 1, 1, 1] + nemo_next_step = 4401 + with self.assertRaises(SystemExit) as context: + nemo_lib.setup_nemo_runlen( + common_env, run_start, model_basis, nemo_step_int, + run_days, run_length, nemo_next_step, None) + self.assertEqual(context.exception.code, error.RESTART_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Last NEMO restarts not at correct time\n' + ' Last completed timestep 4401\n' + ' Expected next step 4402\n' + ' Actual next step 4401\n') diff --git a/Coupled_Drivers/unittests/test_nemo_lib_nl.py b/Coupled_Drivers/unittests/test_nemo_lib_nl.py new file mode 100644 index 00000000..57bd422d --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_lib_nl.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_lib_nl.py + +DESCRIPTION + Test the 'namelist' functions in the NEMO library +''' + +import unittest +import unittest.mock as mock + +import dr_env_lib.nemo_def +import dr_env_lib.ocn_cont_def + +import error +import nemo_lib + + +class TestReadCurrentCycleNLPublic(unittest.TestCase): + ''' + Test the public interface function to read the current namelist cycles + ''' + def setUp(self): + '''Set up the nemo environment variable object''' + self.nemo_envar = {'NEMO_NL': 'nemo_namelist', + 'SI3_NL': 'ice_namelist', + 'TOP_NL': 'top_namelist'} + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_no_models(self, mock_top, mock_si3, mock_nemo): + '''Test that if no models are present we just return nones''' + common_env = {'models': 'no recognised models'} + self.assertEqual( + (None, None, None, None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_not_called() + mock_top.assert_not_called() + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_only(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo only''' + common_env = {'models': 'nemo xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', None, None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_top.assert_not_called() + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_top(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo and top''' + common_env = {'models': 'nemo top xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_top.return_value = ('top_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', None, 'top_rst'), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_top.assert_called_once_with('top_namelist') + mock_si3.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_si3(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo and si3''' + common_env = {'models': 'nemo si3 xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_si3.return_value = ('si3_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', 'si3_rst', None), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_si3.assert_called_once_with('ice_namelist') + mock_top.assert_not_called() + + @mock.patch('nemo_lib._read_current_cycle_nl_nemo') + @mock.patch('nemo_lib._read_current_cycle_nl_si3') + @mock.patch('nemo_lib._read_current_cycle_nl_top') + def test_nemo_si3_top(self, mock_top, mock_si3, mock_nemo): + '''Test for nemo si3 and top''' + common_env = {'models': 'nemo top si3 xios'} + mock_nemo.return_value = ('nemo_rst', 'ln_icebergs') + mock_si3.return_value = ('si3_rst') + mock_top.return_value = ('top_rst') + self.assertEqual( + ('nemo_rst', 'ln_icebergs', 'si3_rst', 'top_rst'), + nemo_lib.read_current_cycle_nl(common_env, self.nemo_envar)) + mock_nemo.assert_called_once_with('nemo_namelist') + mock_si3.assert_called_once_with('ice_namelist') + mock_top.assert_called_once_with('top_namelist') + + +class TestCheckNemoNL(unittest.TestCase): + ''' + Test the existance of the Nemo namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + nemo_envar = {'NEMO_NL': 'namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_nemonl(nemo_envar)) + mock_isfile.assert_called_once_with(nemo_envar['NEMO_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_nemonl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + nemo_envar = {'NEMO_NL': 'namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_nemonl(nemo_envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the nemo namelist file %s\n' % + nemo_envar['NEMO_NL']) + mock_isfile.assert_called_once_with(nemo_envar['NEMO_NL']) + + +class TestCheckSI3NL(unittest.TestCase): + ''' + Test the existance of the SI3 namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + envar = {'SI3_NL': 'ice_namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_si3nl(envar)) + mock_isfile.assert_called_once_with(envar['SI3_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_nemonl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + envar = {'SI3_NL': 'ice_namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_si3nl(envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the SI3 namelist file %s\n' % + envar['SI3_NL']) + mock_isfile.assert_called_once_with(envar['SI3_NL']) + +class TestCheckTopNL(unittest.TestCase): + ''' + Test the existance of the Top namelist file + ''' + @mock.patch('nemo_lib.os.path.isfile') + def test_check_nemonl(self, mock_isfile): + '''Check return zero if the namelist file is present''' + envar = {'TOP_NL': 'top_namelist_cfg'} + mock_isfile.return_value = True + self.assertIsNone(nemo_lib._check_topnl(envar)) + mock_isfile.assert_called_once_with(envar['TOP_NL']) + + @mock.patch('nemo_lib.os.path.isfile') + @mock.patch('nemo_lib.sys.stderr.write') + def test_check_topnl_missing(self, mock_stderr, mock_isfile): + '''Check correct behaviour if the namelist file is missing''' + envar = {'TOP_NL': 'top_namelist_cfg'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as cm: + nemo_lib._check_topnl(envar) + self.assertEqual(cm.exception.code, error.MISSING_DRIVER_FILE_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Can not find the TOP namelist file %s\n' % + envar['TOP_NL']) + mock_isfile.assert_called_once_with(envar['TOP_NL']) + +class TestReadCurrentCycleNLPrivate(unittest.TestCase): + ''' + Test we correctly read from the current cycle namelist. These are the + private functions that carry this out + ''' + def setUp(self): + '''Set up the unit test''' + class DummyReadOcnNL: + '''Dummy for ocn_lib.ReadOcnNamelist''' + def __init__(self, *vars_to_set_to_none): + '''Constructor containing any variables to set to none''' + self.vars_to_set_to_none = vars_to_set_to_none + def read_variables(self, *variables): + '''Check we pass through the correct variables in order, + setting desired ones to None''' + vars_to_return = [] + for var in variables: + vars_to_return.append( + None if var in self.vars_to_set_to_none else var) + return tuple(vars_to_return) + self.DummyNL = DummyReadOcnNL + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + @mock.patch('nemo_lib.string_to_boolean') + def test_read_current_cycle_nl_nemo( + self, mock_str_to_bool, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading NEMO restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'nemo_rst_rmslash' + mock_str_to_bool.return_value = 'bool_icebergs' + self.assertEqual( + ('nemo_rst_rmslash', 'bool_icebergs'), + nemo_lib._read_current_cycle_nl_nemo('namelist_path')) + mock_rmslash.assert_called_once_with('nemo_rst') + mock_str_to_bool.assert_called_once_with('ln_icebergs') + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + def test_read_current_cycle_nl_si3(self, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading SI3 restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'ice_rst_rmslash' + self.assertEqual( + 'ice_rst_rmslash', + nemo_lib._read_current_cycle_nl_si3('namelist_path')) + mock_rmslash.assert_called_once_with(('ice_rst',)) + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.common.remove_trailing_slash') + def test_read_current_cycle_nl_top(self, mock_rmslash, mock_readnl): + '''Test the correct behaviour for reading TOP restart''' + mock_readnl.return_value = self.DummyNL() + mock_rmslash.return_value = 'top_rst_rmslash' + self.assertEqual( + 'top_rst_rmslash', + nemo_lib._read_current_cycle_nl_top('namelist_path')) + mock_rmslash.assert_called_once_with(('top_rst',)) + + + +class TestReadHistoryNl(unittest.TestCase): + ''' + Test the reading of History namelist for nemo + ''' + def setUp(self): + '''Test class for reading of history nl''' + class DummyReadNL: + '''Dummy ocn_lib.ReadOcnNamelist to test interface''' + def __init__(self, nlfilename): + '''Dummy constructor''' + pass + def read_variables(self, *testvar): + '''Verify called with, and return some information''' + assert testvar == ('nemo_first_step', 'nemo_last_step', + 'nemo_step_int', 'nemo_rst_date') + return '1', 'nemo_last_step', '1200', 'nemo_rst_date' + self.namelist = DummyReadNL('namelist_file') + + def test_check_nemo_laststep_digit(self): + '''If nemo last step is a digit, return an integer''' + self.assertEqual(2000, nemo_lib._check_nemo_last_step('2000')) + + def test_check_nemo_laststep_notdigit(self): + '''If nemo last step is not a digit, return 0''' + self.assertEqual(0, nemo_lib._check_nemo_last_step('set_by_system')) + + def test_string_to_boolean_true(self): + '''If the rst_date string is true, return boolean True''' + self.assertTrue(nemo_lib.string_to_boolean('.true.')) + + def test_string_to_boolean_upper_true(self): + '''If the rst_date string is TRUE, return boolean True''' + self.assertTrue(nemo_lib.string_to_boolean('.TRUE.')) + + def test_string_to_boolean_false(self): + '''If the rst_date string is false, return boolean False''' + self.assertFalse(nemo_lib.string_to_boolean('.false.')) + + @mock.patch('nemo_lib.ocn_lib.ReadOcnNamelist') + @mock.patch('nemo_lib.string_to_boolean') + @mock.patch('nemo_lib._check_nemo_last_step') + def test_read_history_nl(self, mock_last_step, mock_str_to_bool, + mock_read_nl): + '''Test the reading of history namelist, and conversions''' + expected_rvalue = (1, 'checked_last', 1200, 'converted_rst') + mock_read_nl.return_value = self.namelist + mock_last_step.return_value = 'checked_last' + mock_str_to_bool.return_value = 'converted_rst' + self.assertEqual( + expected_rvalue, nemo_lib.read_history_nl('namelist_file')) + mock_last_step.assert_called_once_with('nemo_last_step') + mock_str_to_bool.assert_called_once_with('nemo_rst_date') + + +class TestLoadEnvironmentVariables(unittest.TestCase): + ''' + Test the correct reading of environment variables for various submodels + ''' + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_setup_nemo_only(self, mock_load_def, mock_check_nemo, mock_load): + '''Test the correct loading of nemo setup''' + mock_load.return_value = 'nemo_envar' + mock_load_def.return_value = {'L_OCN_PASS_TRC': 'False'} + self.assertEqual( + ({'L_OCN_PASS_TRC': 'False'}, 'nemo xios'), + nemo_lib.load_environment_variables('setup', 'nemo xios')) + mock_load_def.assert_called_once_with( + 'nemo_envar', dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL) + mock_check_nemo.assert_called_once_with({'L_OCN_PASS_TRC': 'False'}) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_final_nemo_only(self, mock_load_def, mock_check_nemo, mock_load): + '''Test the correct loading of nemo final''' + mock_load.return_value = 'nemo_envar' + mock_load_def.return_value = {'L_OCN_PASS_TRC': 'False'} + self.assertEqual( + ({'L_OCN_PASS_TRC': 'False'}, 'nemo xios'), + nemo_lib.load_environment_variables('final', 'nemo xios')) + mock_load_def.assert_called_once_with( + 'nemo_envar', dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL) + mock_check_nemo.assert_called_once_with({'L_OCN_PASS_TRC': 'False'}) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('nemo_lib._check_si3nl') + @mock.patch('nemo_lib._check_topnl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_setup_nemo_top_si3( + self, mock_load_def, mock_check_top, mock_check_si3, + mock_check_nemo, mock_load): + '''Test the correct loading of setup for all 3 models''' + mock_load.return_value = 'nemo_envar' + nemo_load_dict = {'L_OCN_PASS_TRC': 'True'} + si3_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes'} + top_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes', + 'TOP_LOADED': 'yes'} + mock_load_def.side_effect = [nemo_load_dict, si3_load_dict, + top_load_dict] + self.assertEqual( + (top_load_dict, 'nemo xios si3 top'), + nemo_lib.load_environment_variables('setup', 'nemo xios si3')) + mock_load_def.assert_has_calls( + [mock.call('nemo_envar', + dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_INITIAL), + mock.call(nemo_load_dict, + dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_INITIAL), + mock.call(si3_load_dict, + dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_INITIAL)]) + mock_check_nemo.assert_called_once_with(nemo_load_dict) + mock_check_si3.assert_called_once_with(si3_load_dict) + mock_check_top.assert_called_once_with(top_load_dict) + + @mock.patch('nemo_lib.dr_env_lib.env_lib.LoadEnvar') + @mock.patch('nemo_lib._check_nemonl') + @mock.patch('nemo_lib._check_si3nl') + @mock.patch('nemo_lib._check_topnl') + @mock.patch('dr_env_lib.env_lib.load_envar_from_definition') + def test_final_nemo_top_si3( + self, mock_load_def, mock_check_top, mock_check_si3, + mock_check_nemo, mock_load): + '''Test the correct loading of final for all 3 models - Note that + si3 doesnt have any final environment variables''' + mock_load.return_value = 'nemo_envar' + nemo_load_dict = {'L_OCN_PASS_TRC': 'True'} + si3_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes'} + top_load_dict = {'L_OCN_PASS_TRC': 'True', + 'SI3_LOADED': 'yes', + 'TOP_LOADED': 'yes'} + mock_load_def.side_effect = [nemo_load_dict, si3_load_dict, + top_load_dict] + self.assertEqual( + (top_load_dict, 'nemo xios si3 top'), + nemo_lib.load_environment_variables('final', 'nemo xios si3')) + mock_load_def.assert_has_calls( + [mock.call('nemo_envar', + dr_env_lib.nemo_def.NEMO_ENVIRONMENT_VARS_FINAL), + mock.call(nemo_load_dict, + dr_env_lib.ocn_cont_def.SI3_ENVIRONMENT_VARS_FINAL), + mock.call(si3_load_dict, + dr_env_lib.ocn_cont_def.TOP_ENVIRONMENT_VARS_FINAL)]) + mock_check_nemo.assert_called_once_with(nemo_load_dict) + mock_check_si3.assert_called_once_with(si3_load_dict) + mock_check_top.assert_called_once_with(top_load_dict) diff --git a/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py b/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py new file mode 100644 index 00000000..b0fbf84d --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_restart_lib_nl.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_restart_lib_nl.py + +DESCRIPTION + Test the 'namelist' functions in the NEMO restart library +''' + +import unittest +import unittest.mock as mock + +import error +import nemo_restart_lib + +class TestSetupPreviousRestart(unittest.TestCase): + ''' + Test the sourcing from previous work directories in NEMO library + ''' + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_new_run( + self, mock_isfile, mock_setup_crun, mock_setup_nrun, mock_stdout): + '''Test correct calls for NRUN''' + common_env = {'CONTINUE': 'false'} + rvalue = nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', None) + self.assertEqual(rvalue, '.') + mock_stdout.assert_called_once_with('[INFO] New nemo run\n') + mock_setup_nrun.assert_called_once_with( + 'nemo_rst', 'ice_rst', 'top_rst') + mock_setup_crun.assert_not_called() + mock_isfile.assert_not_called() + + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_cont_run( + self, mock_isfile, mock_setup_crun, mock_setup_nrun, mock_stdout): + '''Test correct calls for CRUN''' + common_env = {'CONTINUE': 'true'} + mock_isfile.return_value = True + mock_setup_crun.return_value = 'setup_crun' + rvalue = nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', 'nemo_dump') + self.assertEqual(rvalue, 'setup_crun') + mock_stdout.assert_called_once_with( + '[INFO] Restart data available in NEMO restart ' + 'directory nemo_rst. Restarting from previous task output\n' + '[INFO] Sourcing namelist file from the work ' + 'directory of the previous cycle\n') + mock_setup_nrun.assert_not_called() + mock_setup_crun.assert_called_once() + mock_isfile.assert_called_once_with('nemo_dump') + + @mock.patch('nemo_restart_lib.sys.stderr.write') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_nrun') + @mock.patch('nemo_restart_lib._setup_previous_restart_nl_crun') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_setup_previous_restart_nl_fail(self, mock_isfile, mock_setup_crun, + mock_setup_nrun, mock_stderr): + '''Test the failure mode''' + common_env = {'CONTINUE': 'true'} + mock_isfile.return_value = False + with self.assertRaises(SystemExit) as context: + nemo_restart_lib.setup_previous_restart_nl( + common_env, 'nemo_rst', 'ice_rst', 'top_rst', 'nemo_dump') + self.assertEqual(context.exception.code, error.MISSING_MODEL_FILE_ERROR) + mock_setup_nrun.assert_not_called() + mock_setup_crun.assert_not_called() + mock_isfile.assert_called_once_with('nemo_dump') + mock_stderr.assert_called_once_with( + '[FAIL] No restart data available in NEMO restart ' + 'directory:\n nemo_rst\n') + + @mock.patch('nemo_restart_lib.glob.glob') + @mock.patch('nemo_restart_lib.common.remove_file') + def test_setup_previous_restart_nl_nrun(self, mock_rmfile, mock_glob): + '''Test the removal for restarts for NRUNS so we dont acidentally + pick them up''' + mock_glob.side_effect = [['path1'], ['path2'], ['path3'], ['path4']] + nemo_restart_lib._setup_previous_restart_nl_nrun( + 'nemo_rst', 'ice_rst', 'top_rst') + mock_glob.assert_has_calls([mock.call('nemo_rst/*restart*'), + mock.call('nemo_rst/*trajectory*'), + mock.call('ice_rst/*restart*'), + mock.call('top_rst/*restart*')]) + mock_rmfile.assert_has_calls([mock.call('path1'), + mock.call('path2'), + mock.call('path3'), + mock.call('path4')]) + + @mock.patch('nemo_restart_lib.glob.glob') + @mock.patch('nemo_restart_lib.common.remove_file') + def test_setup_previous_restart_nl_nrun_no_ice( + self, mock_rmfile, mock_glob): + '''Test the removal for restarts for NRUNS so we dont acidentally + pick them up. In this case there is no ice restart set''' + mock_glob.side_effect = [['path1', 'path2'], ['path3']] + nemo_restart_lib._setup_previous_restart_nl_nrun( + 'nemo_rst', None, None) + mock_glob.assert_has_calls([mock.call('nemo_rst/*restart*'), + mock.call('nemo_rst/*trajectory*')]) + mock_rmfile.assert_has_calls([mock.call('path1'), + mock.call('path2'), + mock.call('path3')]) + + def test_setup_previous_restart_nl_crun_cont_from_fail(self): + '''If continue from fail, we run from the current work directory + and so the path returned will be a blank string''' + common_env = {'CONTINUE_FROM_FAIL': 'true'} + self.assertEqual( + '', nemo_restart_lib._setup_previous_restart_nl_crun(common_env)) + + @mock.patch('common.find_previous_workdir') + def test_setup_previous_restart_nl_crun_sub_cycle(self, mock_workdir): + '''Test the Coupled NWP sub cycling mode''' + common_env = {'CONTINUE_FROM_FAIL': 'false', + 'CNWP_SUB_CYCLING': 'True', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point', + 'CYLC_TASK_WORK_DIR': 'work_dir', + 'CYLC_TASK_NAME': 'task_name', + 'CYLC_TASK_PARAM_run': 'param_run'} + mock_workdir.return_value = 'prev_workdir' + self.assertEqual('prev_workdir', + nemo_restart_lib._setup_previous_restart_nl_crun( + common_env)) + mock_workdir.assert_called_once_with( + 'cycle_point', 'work_dir', 'task_name', 'param_run') + + @mock.patch('common.find_previous_workdir') + def test_setup_previous_restart_nl_crun_climate(self, mock_workdir): + '''Test the climate cycling mode''' + common_env = {'CONTINUE_FROM_FAIL': 'false', + 'CNWP_SUB_CYCLING': 'false', + 'CYLC_TASK_CYCLE_POINT': 'cycle_point', + 'CYLC_TASK_WORK_DIR': 'work_dir', + 'CYLC_TASK_NAME': 'task_name'} + mock_workdir.return_value = 'prev_workdir' + self.assertEqual('prev_workdir', + nemo_restart_lib._setup_previous_restart_nl_crun( + common_env)) + mock_workdir.assert_called_once_with('cycle_point', 'work_dir', + 'task_name') diff --git a/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py b/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py new file mode 100644 index 00000000..140c9c87 --- /dev/null +++ b/Coupled_Drivers/unittests/test_nemo_restart_lib_rst.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + test_nemo_restart_lib_rst.py + +DESCRIPTION + Test the 'restart' functions in the NEMO restart library +''' +import unittest +import unittest.mock as mock + +import error +import nemo_restart_lib + +class TestVerifyFixRst(unittest.TestCase): + ''' + Test the verification that the nemo start dump date is consistent with the + start of the cycle + ''' + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_verify_fix_rst_does_match(self, mock_stdout): + '''Test a matching cycle point''' + restart_date = '20210423' + cyclepoint = '20210423T1150Z' + self.assertEqual( + restart_date, nemo_restart_lib.verify_fix_rst( + restart_date, cyclepoint, None)) + mock_stdout.assert_called_once_with( + '[INFO] Validated NEMO restart date\n') + + @mock.patch('nemo_restart_lib.sys.stdout.write') + @mock.patch('nemo_restart_lib.sys.stderr.write') + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.join') + def test_verify_fix_rst_perform_fix( + self, mock_pathjoin, mock_rmfile, mock_findall, mock_lsdir, + mock_stderr, mock_stdout): + '''Test the removal of later restart files''' + expected_msg = '[WARN] The NEMO restart data does not match the ' \ + ' current cycle time\n.' \ + ' Cycle time is 20210423\n' \ + ' NEMO restart time is 20210424\n' \ + '[WARN] Automatically removing NEMO dumps ahead of ' \ + 'the current cycletime, and pick up the dump at ' \ + 'this time\n' + restart_date = '20210424' + cyclepoint = '20210423T1150Z' + restart_regex = r'(icebergs)?.*restart(_trc)?(_\d+)?\.nc' + date_regex = r'\d{8}' + mock_findall.side_effect = [True, False, True, True, + ['20210423'], ['20210422'], ['20210425']] + mock_lsdir.return_value = ['file1', 'file4', 'file2', 'file3'] + mock_pathjoin.return_value = 'joined_rst_path' + self.assertEqual( + '20210423', nemo_restart_lib.verify_fix_rst( + restart_date, cyclepoint, 'nemo_rst')) + mock_findall.assert_has_calls( + [mock.call(restart_regex, 'file1'), + mock.call(restart_regex, 'file4'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file3'), + mock.call(date_regex, 'file1'), + mock.call(date_regex, 'file2'), + mock.call(date_regex, 'file3')]) + mock_pathjoin.assert_called_once_with('nemo_rst', 'file3') + mock_rmfile.assert_called_once_with('joined_rst_path') + mock_stdout.assert_called_once_with(expected_msg) + mock_stderr.assert_called_once_with(expected_msg) + + +class TestCompileNemoRestartFiles(unittest.TestCase): + ''' + Test the determination of NEMO restart files + ''' + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.os.path.join') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_restart_files_latest_dump( + self, mock_isfile, mock_rmfile, mock_pathjoin, + mock_re_findall, mock_listdir): + '''Test behaviour when there are restart files and dump''' + restart_regex = r'.+_\d{8}_restart(_\d+)?\.nc' + # reverse order so they can later be sorted + mock_listdir.return_value = ['file3', 'file2', 'file1'] + mock_re_findall.side_effect = [True, False, True] + mock_pathjoin.return_value = 'latest_nemo_dump' + mock_isfile.return_value = True + self.assertEqual((['file1', 'file3'], 'latest_nemo_dump', 'nemo_rst'), + nemo_restart_lib.compile_nemo_restart_files( + 'nemo_rst')) + mock_listdir.assert_called_once_with('nemo_rst') + mock_re_findall.assert_has_calls([mock.call(restart_regex, 'file3'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file1')]) + mock_pathjoin.assert_called_once_with('nemo_rst', 'file3') + mock_rmfile.assert_has_calls([mock.call('restart.nc'), + mock.call('restart_icebergs.nc'), + mock.call('restart_trc.nc')]) + mock_isfile.assert_called_once_with('latest_nemo_dump') + + + @mock.patch('nemo_restart_lib.os.listdir') + @mock.patch('nemo_restart_lib.re.findall') + @mock.patch('nemo_restart_lib.os.path.join') + @mock.patch('nemo_restart_lib.common.remove_file') + @mock.patch('nemo_restart_lib.os.path.isfile') + def test_no_restart_no_dump( + self, mock_isfile, mock_rmfile, mock_pathjoin, + mock_re_findall, mock_listdir): + '''Test behaviour when there are no restart files and no dumps''' + restart_regex = r'.+_\d{8}_restart(_\d+)?\.nc' + # reverse order so they can later be sorted + mock_listdir.return_value = ['file3', 'file2', 'file1'] + mock_re_findall.side_effect = [False, False, False] + mock_isfile.return_value = False + self.assertEqual(([], 'unset', '.'), + nemo_restart_lib.compile_nemo_restart_files( + 'nemo_rst')) + mock_listdir.assert_called_once_with('nemo_rst') + mock_re_findall.assert_has_calls([mock.call(restart_regex, 'file3'), + mock.call(restart_regex, 'file2'), + mock.call(restart_regex, 'file1')]) + mock_pathjoin.assert_not_called() + mock_rmfile.assert_has_calls([mock.call('restart.nc'), + mock.call('restart_icebergs.nc'), + mock.call('restart_trc.nc')]) + mock_isfile.assert_called_once_with('unset') + +class TestCreateRestartDirecs(unittest.TestCase): + ''' + Test the archiving and creation of NEMO restart directories + ''' + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + def test_crun(self, mock_mkdir, mock_rename): + '''For a CRUN we dont rename or make directories''' + common_env = {'CONTINUE': 'true'} + nemo_restart_lib.create_restart_direcs(None, common_env) + mock_mkdir.assert_not_called() + mock_rename.assert_not_called() + + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_false_options(self, mock_stdout, mock_mkdir, mock_rename, + mock_isdir): + '''Test the option for directly creating a restart dir, continue + is false, and the directory doesnt exist''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['NEMORESTART'] + mock_isdir.return_value = False + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_isdir.assert_called_once_with('NEMORESTART') + mock_mkdir.assert_called_once_with('NEMORESTART') + mock_rename.assert_not_called() + mock_stdout.assert_called_once_with( + '[INFO] Creating NEMO restart directory:\n NEMORESTART\n') + + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_cwd(self, mock_stdout, mock_mkdir, mock_rename, mock_isdir): + '''Test that if the directories are either . or ./ we dont do + anything''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['./', '.'] + mock_isdir.side_effect = [True, True] + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_isdir.assert_has_calls([mock.call('./'), mock.call('.')]) + mock_mkdir.assert_not_called() + mock_stdout.assert_not_called() + mock_rename.assert_not_called() + + @mock.patch('nemo_restart_lib.time.strftime') + @mock.patch('nemo_restart_lib.os.path.isdir') + @mock.patch('nemo_restart_lib.os.rename') + @mock.patch('nemo_restart_lib.os.makedirs') + @mock.patch('nemo_restart_lib.sys.stdout.write') + def test_multi(self, mock_stdout, mock_mkdir, mock_rename, mock_isdir, + mock_time): + '''Test if two restart directorys exist, and one doesnt, that the + two that exist get archived and new ones are created in their place''' + common_env = {'CONTINUE': 'false'} + restart_direcs = ['EXIST_1', 'EXIST_2', 'DONT_EXIST', '.'] + mock_isdir.side_effect = [True, True, False, True] + mock_time.return_value = '202104161924' + nemo_restart_lib.create_restart_direcs(restart_direcs, common_env) + mock_time.assert_called_once_with("%Y%m%d%H%M") + mock_rename.assert_has_calls([ + mock.call('EXIST_1', 'EXIST_1.202104161924'), + mock.call('EXIST_2', 'EXIST_2.202104161924')]) + mock_mkdir.assert_has_calls([mock.call('EXIST_1'), + mock.call('EXIST_2')]) + mock_stdout.assert_has_calls([ + mock.call('[INFO] directory is EXIST_1\n'), + mock.call('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n'), + mock.call('[INFO] directory is EXIST_2\n'), + mock.call('[INFO] This is a New Run. Renaming old NEMO' + ' history directory\n')]) diff --git a/Coupled_Drivers/unittests/test_ocn_lib.py b/Coupled_Drivers/unittests/test_ocn_lib.py new file mode 100644 index 00000000..5d0c01e6 --- /dev/null +++ b/Coupled_Drivers/unittests/test_ocn_lib.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' + +import unittest +import unittest.mock as mock + +from collections import namedtuple +import io +import os +import error +import ocn_lib + +class TestReadOcnNamelist(unittest.TestCase): + ''' + Test the reading of wet model namelists + ''' + def setUp(self): + '''Set up a dummy namelist file''' + self.nl_name = 'test_nl_file' + file_contents = ''' +&TESTNL +nn_it000=1, +nn_itend=2232, +ln_rstdate=.true., +rn_rdt=1200, +dummy=32, +/ + ''' + with open(self.nl_name, 'w') as f_h: + f_h.write(file_contents) + + def tearDown(self): + '''Delete the dummy NL file at the end of thes test''' + try: + os.remove(self.nl_name) + except FileNotFoundError: + pass + + def test_variables_database(self): + '''Test the dictionary containing variables and regular expressions + remains unchaged''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + NamelistVal = namedtuple('NamelistVal', 'regex value') + expected_variables \ + = {'nemo_first_step': NamelistVal(r'nn_it000=(.+),', None), + 'nemo_last_step': NamelistVal(r'nn_itend=(.+),', None), + 'nemo_step_int': NamelistVal(r'rn_rdt=(\d*)', None), + 'nemo_rst_date': NamelistVal(r'ln_rstdate=(.+),', None), + 'nemo_rst': NamelistVal( + r'cn_ocerst_outdir=[\"\'](.*?)[\"\']', None), + 'ice_rst': NamelistVal( + r'cn_icerst_outdir=[\"\'](.*?)[\"\']', None), + 'ln_icebergs': NamelistVal(r'ln_icebergs=(.+),', None), + 'top_rst': NamelistVal( + r'cn_trcrst_outdir=[\"\'](.*?)[\"\']', None)} + self.assertEqual(expected_variables, + read_ocn_namelist_object._variables) + del read_ocn_namelist_object + + def test_read_variables_multiple(self): + '''Test reading in of the variables from the namelist''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + expected_out = ('1', '2232', '.true.', '1200') + self.assertEqual( + expected_out, read_ocn_namelist_object.read_variables( + 'nemo_first_step', 'nemo_last_step', 'nemo_rst_date', + 'nemo_step_int')) + del read_ocn_namelist_object + + def test_read_variables_single(self): + '''Test that reading a single variable returns that variable and not + a tuple''' + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + expected_out = '1' + self.assertEqual(expected_out, + read_ocn_namelist_object.read_variables( + 'nemo_first_step')) + del read_ocn_namelist_object + + @mock.patch('ocn_lib.sys.stderr.write') + def test_read_variables_fail(self, mock_stderr): + read_ocn_namelist_object = ocn_lib.ReadOcnNamelist(self.nl_name) + with self.assertRaises(SystemExit) as context: + read_ocn_namelist_object.read_variables('invalid_var') + self.assertEqual(context.exception.code, + error.MISSING_FILE_CONTENTS_ERROR) + mock_stderr.assert_called_once_with( + '[FAIL] Unable to determine how to read the variable invalid_var' + ' from file %s\n' % (self.nl_name)) + del read_ocn_namelist_object + +class TestGetNemoNprocStr(unittest.TestCase): + ''' + Unit tests for determining the number of zeros in the unrebuilt NEMO + restart files + ''' + @mock.patch('ocn_lib.glob.glob') + def test_index_error_exception(self, mock_glob): + '''Test that if the index error is thrown, we have a returned length + of zero''' + mock_glob.side_effect = IndexError + self.assertEqual(ocn_lib._get_nemo_nproc_str(), 0) + + @mock.patch('ocn_lib.glob.glob') + def test_default_restart_directory(self, mock_glob): + '''Test correct behaviour with default dictionary''' + mock_glob.return_value = ['oce_2021_restart_000.nc'] + self.assertEqual(ocn_lib._get_nemo_nproc_str(), 3) + mock_glob.assert_called_with('./*_[0-9]*_restart_*0.nc') + + @mock.patch('ocn_lib.glob.glob') + def test_non_default_restart_directory(self, mock_glob): + '''Test correct behaviour with non default directory''' + mock_glob.return_value = ['oce_2021_restart_0.nc'] + self.assertEqual(ocn_lib._get_nemo_nproc_str('custom_dir'), 1) + mock_glob.assert_called_with('custom_dir/*_[0-9]*_restart_*0.nc') + + +class TestSetupMultipleRestart(unittest.TestCase): + ''' + Unit tests for setting up of multiple restart files + ''' + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_two_restarts(self, mock_slink, mock_isfile, mock_rmfile): + '''Test the correct linking of two restart files. Two nemo processors + but a nproc string length of 4''' + mock_isfile.side_effect = [True, True] + ocn_lib._setup_multiple_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 2, 4) + mock_rmfile.assert_has_calls([mock.call('rst_link_dir/lnname_0000.nc'), + mock.call('rst_link_dir/lnname_0001.nc')]) + mock_isfile.assert_has_calls([mock.call('init_dir/fname_0000.nc'), + mock.call('init_dir/fname_0001.nc')]) + mock_slink.assert_has_calls([mock.call('init_dir/fname_0000.nc', + 'rst_link_dir/lnname_0000.nc'), + mock.call('init_dir/fname_0001.nc', + 'rst_link_dir/lnname_0001.nc')]) + + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_no_file(self, mock_slink, mock_isfile, mock_rmfile): + '''Test that if a file isn't found it isn't linked''' + mock_isfile.return_value = False + ocn_lib._setup_multiple_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 1, 4) + mock_rmfile.assert_called_once_with('rst_link_dir/lnname_0000.nc') + mock_isfile.assert_called_once_with('init_dir/fname_0000.nc') + mock_slink.assert_not_called() + + +class TestSetupSingleRestart(unittest.TestCase): + ''' + Unit tests for setting up a single restart file + ''' + @mock.patch('ocn_lib.sys.stdout.write') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_single_rst(self, mock_slink, mock_isfile, mock_rmfile, + mock_stdout): + '''Test the correct linking of a single restart file''' + mock_isfile.return_value = True + ocn_lib._setup_single_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_rmfile.assert_called_once_with('rst_link_dir/lnname.nc') + mock_isfile.assert_called_once_with('init_dir/fname.nc') + mock_slink.assert_called_once_with('init_dir/fname.nc', + 'rst_link_dir/lnname.nc') + mock_stdout.assert_has_calls( + [mock.call('[INFO] No MODEL sub-PE restarts found\n'), + mock.call('[INFO] Using rebuilt MODEL restart file' + ' init_dir/fname.nc\n')]) + + @mock.patch('ocn_lib.sys.stdout.write') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_single_rst_no_file(self, mock_slink, mock_isfile, mock_rmfile, + mock_stdout): + '''Test that if the file is not found, no link attempted''' + mock_isfile.return_value = False + ocn_lib._setup_single_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_rmfile.assert_called_once_with('rst_link_dir/lnname.nc') + mock_isfile.assert_called_once_with('init_dir/fname.nc') + mock_slink.assert_not_called() + mock_stdout.assert_called_once_with( + '[INFO] No MODEL sub-PE restarts found\n') + +class TestSetupRestart(unittest.TestCase): + ''' + Test the top level of the setup restart functionality + ''' + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib._setup_multiple_restart') + @mock.patch('ocn_lib._setup_single_restart') + def test_setup_restart_multiple(self, mock_single, mock_multiple, + mock_nproc, mock_linkdir): + '''Setup the restart for multiple restart files''' + mock_nproc.return_value = 2 + ocn_lib.setup_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', + 'MODEL') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_multiple.assert_called_once_with( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', 2) + mock_single.assert_not_called() + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib._setup_multiple_restart') + @mock.patch('ocn_lib._setup_single_restart') + def test_setup_restart_single(self, mock_single, mock_multiple, + mock_nproc, mock_linkdir): + '''Setup the restart for single restart files''' + mock_nproc.return_value = 0 + ocn_lib.setup_restart( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'nemo_nproc', + 'MODEL') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_single.assert_called_once_with( + 'init_dir', 'rst_link_dir', 'fname', 'lnname', 'MODEL') + mock_multiple.assert_not_called() + +class TestSetupNRUN(unittest.TestCase): + ''' + Test setting up of restart files for an NRUN + ''' + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib.sys.stdout.write') + def test_setup_nrun_unset_start_envar(self, mock_stdout, mock_linkdir): + '''Test a value for the START environment variable which evaluates to + false''' + self.assertEqual(ocn_lib.setup_nrun( + False, 'rst_link_dir', None, None, 'my_warning'), '.false.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_stdout.assert_called_once_with('[WARN] my_warning\n') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.sys.stderr.write') + def test_setup_nrun_no_restart( + self, mock_stderr, mock_isfile, mock_nproc, mock_linkdir): + '''Check we exit correctly if the restart file is not found''' + mock_nproc.return_value = 0 + mock_isfile.side_effect = [False, False] + with self.assertRaises(SystemExit) as context: + ocn_lib.setup_nrun('start', 'rst_link_dir', None, None, None) + self.assertEqual(context.exception.code, + error.MISSING_MODEL_FILE_ERROR) + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_stderr.assert_called_once_with('[FAIL] file start not found\n') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.os.symlink') + def test_setup_nrun_single_file( + self, mock_slink, mock_isfile, mock_nproc, mock_linkdir): + '''Check the linking of a single restart file for NRUN''' + mock_nproc.return_value = 0 + mock_isfile.return_value = True + self.assertEqual(ocn_lib.setup_nrun( + 'start', 'rst_link_dir', 'restart', 'init', None), '.true.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_nproc.assert_called_once_with('init') + mock_slink.assert_called_once_with('start', 'rst_link_dir/restart.nc') + + @mock.patch('ocn_lib._create_restart_link_dir') + @mock.patch('ocn_lib._get_nemo_nproc_str') + @mock.patch('ocn_lib.os.path.isfile') + @mock.patch('ocn_lib.glob.glob') + @mock.patch('ocn_lib.common.remove_file') + @mock.patch('ocn_lib.os.symlink') + def test_setup_nrun_multiple_file(self, mock_slink, mock_rmfile, mock_glob, + mock_isfile, mock_nproc, mock_linkdir): + '''Check the linking of multiple restart files for NRUN''' + mock_nproc.return_value = 4 + mock_isfile.side_effect = [False, True] + mock_glob.return_value = ['start_0000.nc', 'start_0001.nc'] + self.assertEqual(ocn_lib.setup_nrun( + 'start', 'rst_link_dir', 'restart', 'init', None), '.true.') + mock_linkdir.assert_called_once_with('rst_link_dir') + mock_nproc.assert_called_once_with('init') + mock_isfile.assert_has_calls([mock.call('start'), + mock.call('start_0000.nc')]) + mock_glob.assert_called_once_with('start_????.nc') + mock_rmfile.assert_has_calls( + [mock.call('rst_link_dir/restart_0000.nc'), + mock.call('rst_link_dir/restart_0001.nc')]) + mock_slink.assert_has_calls( + [mock.call('start_0000.nc', 'rst_link_dir/restart_0000.nc'), + mock.call('start_0001.nc', 'rst_link_dir/restart_0001.nc')]) + +class TestCreateRestartLinkDir(unittest.TestCase): + ''' + Test the functionaility of creating restart directory + ''' + @mock.patch('ocn_lib.os.mkdir') + def test_create_linkdir_wd(self, mock_mkdir): + '''Test that nothing happens if the restart dir is the current + working dir '.' ''' + self.assertIsNone(ocn_lib._create_restart_link_dir('.')) + mock_mkdir.assert_not_called() + + @mock.patch('ocn_lib.os.mkdir') + def test_create_linkdir(self, mock_mkdir): + '''Test that we call os.mkdir when the directory doesnt exist''' + self.assertIsNone(ocn_lib._create_restart_link_dir('restart_dir')) + mock_mkdir.assert_called_once_with('restart_dir') + + @mock.patch('ocn_lib.os.mkdir') + @mock.patch('ocn_lib.sys.stdout.write') + def test_create_linkdir_existing_folder(self, mock_stdout, mock_mkdir): + '''Test that a FileExistsError is captured if dictionary can't be + created''' + mock_mkdir.side_effect = FileExistsError + self.assertIsNone(ocn_lib._create_restart_link_dir('restart_dir')) + mock_stdout.assert_called_once_with( + '[INFO] Restart link subdirectory restart_dir exists\n') diff --git a/Coupled_Drivers/unittests/test_update_nemo_nl.py b/Coupled_Drivers/unittests/test_update_nemo_nl.py new file mode 100644 index 00000000..562f7d8f --- /dev/null +++ b/Coupled_Drivers/unittests/test_update_nemo_nl.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +''' +import os +import unittest +import unittest.mock as mock + +import update_nemo_nl + +class TestUpdateSI3NL(unittest.TestCase): + ''' + Test the update SI3 NL file + ''' + def setUp(self): + '''Set up the unit test''' + # The internal variables dictionary + self.variables = {'cn_icerst_indir': "'ocean_link_directory'"} + self.nemo_envar = {'SI3_NL': 'si3_namelist', + 'RST_LINK_DIR': 'ocean_link_directory'} + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_si3(self, mock_move, mock_update): + '''Test the correct behaviour for updating SI3 namelist''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_si3_nl(self.nemo_envar) + mock_update.assert_called_once_with('si3_namelist', self.variables) + mock_move.assert_called_once_with('my_nl_file.tmp', 'si3_namelist') + +class TestUpdateTopNL(unittest.TestCase): + ''' + Test the update Top NL file + ''' + def setUp(self): + '''Set up the variables dictionary''' + # The internal variables dictionary + self.variables = {'ln_rsttr': 'my_ln_restart', + 'nn_rsttr': 'my_restart_ctl', + 'ln_trcdta': 'my_ln_trcdta', + 'cn_trcrst_indir': "'ocean_link_directory'"} + self.nemo_envar = {'TOP_NL': 'top_namelist', + 'RST_LINK_DIR': 'ocean_link_directory'} + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_top(self, mock_move, mock_update): + '''Test the correct behaviour for updating top namelist''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_top_nl(self.nemo_envar, 'my_ln_restart', + 'my_restart_ctl', 'my_ln_trcdta') + mock_update.assert_called_once_with('top_namelist', self.variables) + mock_move.assert_called_once_with('my_nl_file.tmp', 'top_namelist') + + +class TestUpdateNemoNL(unittest.TestCase): + ''' + Test the update nemo nl file, with NEMO3.6 and NEMO4 behaviour + ''' + def setUp(self): + '''Set up the variables dictionary''' + # The internal variable dictionaries + self.variables_4 = {'cn_exp': "'my_runido'", + 'ln_rstart': 'my_restart', + 'nn_rstctl': 'my_ctl', + 'nn_it000': 'my_next_step', + 'nn_itend': 'my_final_step', + 'nn_date0': 'my_ndate0', + 'nn_leapy': 'my_leapy', + 'jpni': 'my_iproc', + 'jpnj': 'my_jproc', + 'nn_cpl_river': 'my_cpl_river', + 'cn_ocerst_indir': "'ocean_link_directory'"} + self.variables_36 = {**self.variables_4, **{'jpnij': 'my_jpnij'}} + + # Common envar to be passed in + self.common_envar = {'RUNID': 'my_runid', + 'CPL_RIVER_COUNT': 'my_cpl_river'} + + # Nemo environment variables to be passed in + self.nemo_envar_4 = {'NEMO_VERSION': 400, + 'NEMO_IPROC': 'my_iproc', + 'NEMO_JPROC': 'my_jproc', + 'NEMO_NL': 'my_nl_file', + 'RST_LINK_DIR': 'ocean_link_directory'} + # Sort out for nemo 3.6 + self.nemo_envar_36 = {**self.nemo_envar_4, **{'NEMO_NPROC': 'my_jpnij'}} + self.nemo_envar_36['NEMO_VERSION'] = 360 + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_nl_nemo_4(self, mock_move, mock_update): + '''Test the correct behaviour for NEMO4''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_nl(self.common_envar, self.nemo_envar_4, + 'my_restart', 'my_ctl', 'my_next_step', + 'my_final_step', 'my_ndate0', 'my_leapy') + mock_update.assert_called_with('my_nl_file', self.variables_4) + mock_move.assert_called_with('my_nl_file.tmp', 'my_nl_file') + + @mock.patch('update_nemo_nl.do_update_nl_files') + @mock.patch('update_nemo_nl.shutil.move') + def test_update_nl_nemo_36(self, mock_move, mock_update): + '''Test the correct behaviour for NEMO3.6''' + mock_update.return_value = 'my_nl_file.tmp' + update_nemo_nl.update_nl(self.common_envar, self.nemo_envar_36, + 'my_restart', 'my_ctl', 'my_next_step', + 'my_final_step', 'my_ndate0', 'my_leapy') + mock_update.assert_called_with('my_nl_file', self.variables_36) + mock_move.assert_called_with('my_nl_file.tmp', 'my_nl_file') + + +class TestDoUpdateNLFiles(unittest.TestCase): + ''' + Test the updating of namelist files + ''' + def setUp(self): + '''Create a dummy namelist file to update''' + self.nl_name = 'dummy_nl_cfg' + self.nl_swap_name = 'dummy_nl_cfg.tmp' + # Create our input file + file_contents = '''&namrun +swap1=original_value1, +ln_mskland=.true., +/ +&namzgr +swap2='original_value2', +swap3=original_value3, +swap4=original_value4, +ln_zps=.true., +/ +''' + with open(self.nl_name, 'w') as fh: + fh.write(file_contents) + + # Expected contents after swap + self.expected_out = '''&namrun +swap1=new_value1, +ln_mskland=.true., +/ +&namzgr +swap2=.true., +swap3=.false., +swap4=4.5, +ln_zps=.true., +/ +''' + + def tearDown(self): + '''Remove any files created in the unit test''' + for filename in [self.nl_name, self.nl_swap_name]: + try: + os.remove(filename) + except FileNotFoundError: + pass + + def test_update_nl_files(self): + '''Test the updating function''' + variables = {'swap1': 'new_value1', + 'swap2': '.true.', + 'swap3': '.false.', + 'swap4': 4.5} + rvalue = update_nemo_nl.do_update_nl_files(self.nl_name, variables) + self.assertEqual(rvalue, self.nl_swap_name) + with open(self.nl_swap_name, 'r') as fh: + for line, expected_line in zip(fh.readlines(), + self.expected_out.split('\n')): + self.assertEqual(line.rstrip(), expected_line) + diff --git a/Coupled_Drivers/update_nemo_nl.py b/Coupled_Drivers/update_nemo_nl.py new file mode 100644 index 00000000..5229d9ee --- /dev/null +++ b/Coupled_Drivers/update_nemo_nl.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +''' +*****************************COPYRIGHT****************************** + (C) Crown copyright 2021 Met Office. All rights reserved. + + Use, duplication or disclosure of this code is subject to the restrictions + as set forth in the licence. If no licence has been raised with this copy + of the code, the use, duplication or disclosure of it is strictly + prohibited. Permission to do so must first be obtained in writing from the + Met Office Information Asset Owner at the following address: + + Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom +*****************************COPYRIGHT****************************** +NAME + update_nemo_nl.py + +DESCRIPTION + Update the NEMO namelists. Valid for NEMO 3.6 and NEMO 4.0. The interface + function is update_nl +''' + +import shutil +import common + +def do_update_nl_files(namelist_filename, variables): + ''' + Do the replacement of the variables in the namelist file and + return the filename of the new file + ''' + updated_nl_file = '%s.tmp' % namelist_filename + _ = common.remove_file(updated_nl_file) + with common.open_text_file(updated_nl_file, 'w') as new_fh: + with common.open_text_file(namelist_filename, 'r') as old_fh: + for line in old_fh.readlines(): + if '=' in line: + varname = line.split('=')[0] + if varname in variables.keys(): + newline = '%s=%s,\n' % (varname, variables[varname]) + new_fh.write(newline) + else: + new_fh.write(line) + else: + new_fh.write(line) + return updated_nl_file + +def update_si3_nl(nemo_envar): + ''' + Update the SI3 namelist + ''' + variables = {'cn_icerst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + updated_nl_filename = do_update_nl_files(nemo_envar['SI3_NL'], variables) + shutil.move(updated_nl_filename, nemo_envar['SI3_NL']) + +def update_top_nl(nemo_envar, ln_restart, restart_ctl, ln_trcdta): + ''' + Update the Top namelist + ''' + variables = {'ln_rsttr': ln_restart, + 'nn_rsttr': restart_ctl, + 'ln_trcdta': ln_trcdta, + 'cn_trcrst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + updated_nl_filename = do_update_nl_files(nemo_envar['TOP_NL'], variables) + shutil.move(updated_nl_filename, nemo_envar['TOP_NL']) + + +def update_nl(common_envar, nemo_envar, ln_restart, restart_ctl, nemo_next_step, + nemo_final_step, nemo_ndate0, nleapy): + ''' + Update the NEMO namelist + ''' + # variables to replace for NEMO3.6 and NEMO4.0 + # The run ID needs to appear in quotes in the namelist file and takes the + # letter 'o' as a suffix (cn_exp) + variables = {'cn_exp': "'%so'" % common_envar['RUNID'], + 'ln_rstart': ln_restart, + 'nn_rstctl': restart_ctl, + 'nn_it000': nemo_next_step, + 'nn_itend': nemo_final_step, + 'nn_date0': nemo_ndate0, + 'nn_leapy': nleapy, + 'jpni': nemo_envar['NEMO_IPROC'], + 'jpnj': nemo_envar['NEMO_JPROC'], + 'nn_cpl_river': common_envar['CPL_RIVER_COUNT'], + 'cn_ocerst_indir': "'%s'" % nemo_envar['RST_LINK_DIR']} + # add nemo3.6 only variable + if int(nemo_envar['NEMO_VERSION']) < 400: + variables['jpnij'] = nemo_envar['NEMO_NPROC'] + + updated_nl_filename = do_update_nl_files(nemo_envar['NEMO_NL'], + variables) + shutil.move(updated_nl_filename, nemo_envar['NEMO_NL']) diff --git a/rose-stem/site/meto_azspice.cylc b/rose-stem/site/meto_azspice.cylc index 1e0481e3..579042a7 100644 --- a/rose-stem/site/meto_azspice.cylc +++ b/rose-stem/site/meto_azspice.cylc @@ -18,7 +18,7 @@ [[[environment]]] PYTHON_ENVIRONMENT = """ echo *** Linux Python Environment Begin *** -module load scitools scitools/default-current +module load scitools/default-current module load um_tools echo $(module list) echo *** Linux Python Environment End *** @@ -40,7 +40,7 @@ echo *** Linux Python Environment End *** {% if USE_LINUX_SERVER %} [[[directives]]] --mem=10G - {% endif %} + {% endif %} {% endif %} [[DRIVERS_TESTS_RESOURCE]] diff --git a/rose-stem/site/meto_ex1a.cylc b/rose-stem/site/meto_ex1a.cylc index f79a8e60..567b5631 100644 --- a/rose-stem/site/meto_ex1a.cylc +++ b/rose-stem/site/meto_ex1a.cylc @@ -1,4 +1,4 @@ -{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools;' %} +{% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools; module load um_tools;' %} [[HPC_RESOURCE]] env-script = "eval $(rose task-env)" @@ -94,8 +94,8 @@ DATA_INTENSITY = true COMPLEXITY = false TOTAL_POWER_CONSUMPTION_NEEDED = 4.8 - NODES_IN_HPC_NEEDED = 12932 - [[[directives]]] + NODES_IN_HPC_NEEDED = 12932 + [[[directives]]] -l select = '{{ATMOS_NODES}}:ncpus=256:mpiprocs={{ATMOS_MPIPROC_PN}}+{{OCEAN_NODES}}:ncpus=256:mpiprocs={{OCEAN_PPN}}+{{XIOS_NODES}}:ncpus=256:mpiprocs={{XIOS_NPROC}}' [[RECON_RESOURCE]]