diff --git a/CIME/BuildTools/configure.py b/CIME/BuildTools/configure.py index 6e77a1a15c5..2e833735472 100755 --- a/CIME/BuildTools/configure.py +++ b/CIME/BuildTools/configure.py @@ -15,20 +15,22 @@ COMPILER, MPILIB, and DEBUG, respectively. """ -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import shutil + +from CIME.build import CmakeTmpBuildDir from CIME.utils import ( + copy_local_macros_to_dir, expect, - safe_copy, get_model, get_src_root, + safe_copy, stringify_bool, - copy_local_macros_to_dir, ) from CIME.XML.env_mach_specific import EnvMachSpecific from CIME.XML.files import Files -from CIME.build import CmakeTmpBuildDir - -import shutil, glob logger = logging.getLogger(__name__) diff --git a/CIME/Servers/ftp.py b/CIME/Servers/ftp.py index 209525a2bac..12c50e18767 100644 --- a/CIME/Servers/ftp.py +++ b/CIME/Servers/ftp.py @@ -1,15 +1,20 @@ """ FTP Server class. Interact with a server using FTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * -from CIME.Servers.generic_server import GenericServer -from CIME.utils import Timeout +import logging +import os +import socket from ftplib import FTP as FTPpy from ftplib import all_errors as all_ftp_errors -import socket + +from CIME.Servers.generic_server import GenericServer +from CIME.utils import Timeout, expect logger = logging.getLogger(__name__) + + # I think that multiple inheritence would be useful here, but I couldnt make it work # in a py2/3 compatible way. class FTP(GenericServer): diff --git a/CIME/Servers/generic_server.py b/CIME/Servers/generic_server.py index 537df181324..61fcef21b8f 100644 --- a/CIME/Servers/generic_server.py +++ b/CIME/Servers/generic_server.py @@ -3,9 +3,10 @@ to make sure that specific server classes maintain a consistant argument list and functionality so that they are interchangable objects """ + # pylint: disable=unused-argument -from CIME.XML.standard_module_setup import * +import logging from socket import _GLOBAL_DEFAULT_TIMEOUT logger = logging.getLogger(__name__) diff --git a/CIME/Servers/gftp.py b/CIME/Servers/gftp.py index f23943b583a..fea2a8a0915 100644 --- a/CIME/Servers/gftp.py +++ b/CIME/Servers/gftp.py @@ -1,8 +1,11 @@ """ GridFTP Server class. Interact with a server using GridFTP protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer from CIME.utils import run_cmd diff --git a/CIME/Servers/svn.py b/CIME/Servers/svn.py index 7e06ab310d4..b407683ae07 100644 --- a/CIME/Servers/svn.py +++ b/CIME/Servers/svn.py @@ -1,9 +1,13 @@ """ SVN Server class. Interact with a server using SVN protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/Servers/wget.py b/CIME/Servers/wget.py index fde411460e8..583e0972bc4 100644 --- a/CIME/Servers/wget.py +++ b/CIME/Servers/wget.py @@ -1,9 +1,13 @@ """ WGET Server class. Interact with a server using WGET protocol """ + # pylint: disable=super-init-not-called -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.Servers.generic_server import GenericServer +from CIME.utils import run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/dae.py b/CIME/SystemTests/dae.py index 175254d2d1b..e9472678de2 100644 --- a/CIME/SystemTests/dae.py +++ b/CIME/SystemTests/dae.py @@ -6,14 +6,15 @@ """ -import os.path -import logging import glob import gzip +import logging +import os.path from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect + ############################################################################### class DAE(SystemTestsCompareTwo): ############################################################################### diff --git a/CIME/SystemTests/eri.py b/CIME/SystemTests/eri.py index 597cfe4c709..151f07d55ac 100644 --- a/CIME/SystemTests/eri.py +++ b/CIME/SystemTests/eri.py @@ -2,11 +2,14 @@ CIME ERI test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import glob +import logging +import os +import shutil from stat import S_ISDIR, ST_CTIME, ST_MODE -import shutil, glob, os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erio.py b/CIME/SystemTests/erio.py index a1e7b041cc6..0ded2f0d273 100644 --- a/CIME/SystemTests/erio.py +++ b/CIME/SystemTests/erio.py @@ -3,8 +3,11 @@ This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erp.py b/CIME/SystemTests/erp.py index aeab1b0020b..b0fa12696c0 100644 --- a/CIME/SystemTests/erp.py +++ b/CIME/SystemTests/erp.py @@ -8,7 +8,8 @@ (2) Do a restart test with half the number of tasks and threads (suffix rest) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.restart_tests import RestartTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/err.py b/CIME/SystemTests/err.py index ff5a2e4a4f7..755381e552c 100644 --- a/CIME/SystemTests/err.py +++ b/CIME/SystemTests/err.py @@ -3,10 +3,12 @@ ERR tests short term archiving and restart capabilities """ -import glob, os -from CIME.XML.standard_module_setup import * +import glob +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.utils import safe_copy, ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/erri.py b/CIME/SystemTests/erri.py index 7851bd4bb66..e3453aeaccb 100644 --- a/CIME/SystemTests/erri.py +++ b/CIME/SystemTests/erri.py @@ -3,10 +3,13 @@ ERRI tests short term archiving and restart capabilities with "incomplete" (unzipped) log files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.err import ERR +import glob +import gzip +import logging +import os +import shutil -import shutil, glob, gzip +from CIME.SystemTests.err import ERR logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ers.py b/CIME/SystemTests/ers.py index 0e93d8afa1c..b39bf71aef1 100644 --- a/CIME/SystemTests/ers.py +++ b/CIME/SystemTests/ers.py @@ -1,9 +1,13 @@ """ CIME restart test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + import glob +import logging +import os + +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ert.py b/CIME/SystemTests/ert.py index b912f7248b7..5a298336e0f 100644 --- a/CIME/SystemTests/ert.py +++ b/CIME/SystemTests/ert.py @@ -3,7 +3,8 @@ Exact restart from startup, default 2 month + 1 month """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/funit.py b/CIME/SystemTests/funit.py index 7681a6200d6..2a366e6b3d6 100644 --- a/CIME/SystemTests/funit.py +++ b/CIME/SystemTests/funit.py @@ -2,12 +2,15 @@ CIME FUNIT test. This class inherits from SystemTestsCommon. It runs the fortran unit tests; grid and compset are ignored. """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon + +import logging +import os + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import get_cime_root -from CIME.test_status import * +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS +from CIME.utils import expect, get_cime_root, run_cmd logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/hommebaseclass.py b/CIME/SystemTests/hommebaseclass.py index d275553a545..dd3183c5ad6 100644 --- a/CIME/SystemTests/hommebaseclass.py +++ b/CIME/SystemTests/hommebaseclass.py @@ -2,14 +2,22 @@ CIME HOMME test. This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import logging +import os +import shutil + from CIME.build import post_build from CIME.status import append_testlog -from CIME.utils import SharedArea, new_lid, gzip_existing_file -from CIME.test_status import * - -import shutil +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.test_status import BASELINE_PHASE, GENERATE_PHASE, TEST_PASS_STATUS +from CIME.utils import ( + SharedArea, + expect, + gzip_existing_file, + new_lid, + run_cmd, + run_cmd_no_fail, +) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/icp.py b/CIME/SystemTests/icp.py index 8d8c5e0ea59..c5357784671 100644 --- a/CIME/SystemTests/icp.py +++ b/CIME/SystemTests/icp.py @@ -1,7 +1,7 @@ """ CIME ICP test This class inherits from SystemTestsCommon """ -from CIME.XML.standard_module_setup import * + from CIME.SystemTests.system_tests_common import SystemTestsCommon diff --git a/CIME/SystemTests/irt.py b/CIME/SystemTests/irt.py index 633893adcd0..419dfd8164a 100644 --- a/CIME/SystemTests/irt.py +++ b/CIME/SystemTests/irt.py @@ -11,9 +11,11 @@ """ +import logging +import os + from CIME.SystemTests.restart_tests import RestartTest -from CIME.XML.standard_module_setup import * -from CIME.utils import ls_sorted_by_fname +from CIME.utils import expect, ls_sorted_by_fname logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ldsta.py b/CIME/SystemTests/ldsta.py index a5f7c9196d5..4dd96fb8bae 100644 --- a/CIME/SystemTests/ldsta.py +++ b/CIME/SystemTests/ldsta.py @@ -4,19 +4,20 @@ The test verifies the archive directory contains the expected files """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.utils import expect -from CIME.date import get_file_date - import datetime import glob +import logging import os import random import shutil +from CIME.date import get_file_date +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.utils import expect + logger = logging.getLogger(__name__) + # datetime objects can't be used anywhere else def _date_to_datetime(date_obj): return datetime.datetime( diff --git a/CIME/SystemTests/mcc.py b/CIME/SystemTests/mcc.py index a4b839cf1e9..f46030cdffd 100644 --- a/CIME/SystemTests/mcc.py +++ b/CIME/SystemTests/mcc.py @@ -4,7 +4,9 @@ This does two runs: In the first we run a three member ensemble using the MULTI_DRIVER capability, then we run a second single instance case and compare """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 60f863a2670..852a460d248 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -6,23 +6,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging +import os from shutil import copytree -from CIME import test_status -from CIME import utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + +from CIME import test_status, utils from CIME.case.case_setup import case_setup -from CIME.XML.machines import Machines from CIME.config import ConfigBase -from CIME.SystemTests import test_mods from CIME.namelist import Namelist - -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error +from CIME.status import append_testlog +from CIME.SystemTests import test_mods +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.XML.machines import Machines version = evv4esm.__version_info__ diff --git a/CIME/SystemTests/nck.py b/CIME/SystemTests/nck.py index f75a2914215..9f17b28c09d 100644 --- a/CIME/SystemTests/nck.py +++ b/CIME/SystemTests/nck.py @@ -8,7 +8,8 @@ Lay all of the components out sequentially """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/ncr.py b/CIME/SystemTests/ncr.py index f0de168ac13..39b16f25ea2 100644 --- a/CIME/SystemTests/ncr.py +++ b/CIME/SystemTests/ncr.py @@ -8,7 +8,9 @@ NOTE: This is currently untested, and may not be working properly """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/nodefail.py b/CIME/SystemTests/nodefail.py index d975cfc5bfd..559f49c690b 100644 --- a/CIME/SystemTests/nodefail.py +++ b/CIME/SystemTests/nodefail.py @@ -1,7 +1,10 @@ """ CIME restart upon failed node test. """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME.SystemTests.ers import ERS from CIME.utils import get_model diff --git a/CIME/SystemTests/pea.py b/CIME/SystemTests/pea.py index 4fb3a4569ca..c7534dd8340 100644 --- a/CIME/SystemTests/pea.py +++ b/CIME/SystemTests/pea.py @@ -6,8 +6,10 @@ (2) do a run with mpi-serial (suffix mpi-serial) """ +import logging +import os + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pem.py b/CIME/SystemTests/pem.py index d98f9e4a2c7..d26108d8197 100644 --- a/CIME/SystemTests/pem.py +++ b/CIME/SystemTests/pem.py @@ -8,7 +8,8 @@ (2) Run with half the number of tasks (suffix modpes) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pet.py b/CIME/SystemTests/pet.py index 161dac4d536..35e905f2399 100644 --- a/CIME/SystemTests/pet.py +++ b/CIME/SystemTests/pet.py @@ -6,7 +6,8 @@ (2) do another initial run with nthrds=1 for all components (suffix: single_thread) """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pfs.py b/CIME/SystemTests/pfs.py index ed61d204e8a..ffd1ed46d3b 100644 --- a/CIME/SystemTests/pfs.py +++ b/CIME/SystemTests/pfs.py @@ -4,7 +4,8 @@ 20 days performance test, no restart files written """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index 07ac1f4cb08..ed27bc6d641 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -9,30 +9,27 @@ from __future__ import division +import json +import logging import os import re -import json import shutil -import logging - from collections import OrderedDict from shutil import copytree -import pandas as pd +import evv4esm # pylint: disable=import-error import numpy as np - +import pandas as pd +from evv4esm.__main__ import main as evv # pylint: disable=import-error +from evv4esm.extensions import pg # pylint: disable=import-error import CIME.test_status import CIME.utils +from CIME.case.case_setup import case_setup from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.extensions import pg # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/pre.py b/CIME/SystemTests/pre.py index 23547d46430..c5245364eb1 100644 --- a/CIME/SystemTests/pre.py +++ b/CIME/SystemTests/pre.py @@ -6,13 +6,14 @@ Test requires DESP component to function correctly. """ -import os.path -import logging import glob +import logging +import os.path +from CIME.hist_utils import cprnc from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.utils import expect -from CIME.hist_utils import cprnc + ############################################################################### class PRE(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/restart_tests.py b/CIME/SystemTests/restart_tests.py index ffc50270ce2..6883e95baa0 100644 --- a/CIME/SystemTests/restart_tests.py +++ b/CIME/SystemTests/restart_tests.py @@ -2,8 +2,10 @@ Abstract class for restart tests """ +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo -from CIME.XML.standard_module_setup import * +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/reuseinitfiles.py b/CIME/SystemTests/reuseinitfiles.py index 76d8bb0522e..98cf8216380 100644 --- a/CIME/SystemTests/reuseinitfiles.py +++ b/CIME/SystemTests/reuseinitfiles.py @@ -14,9 +14,10 @@ import os import shutil -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo + from CIME.SystemTests.system_tests_common import INIT_GENERATED_FILES_DIRNAME +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import expect class REUSEINITFILES(SystemTestsCompareTwo): diff --git a/CIME/SystemTests/seq.py b/CIME/SystemTests/seq.py index 7413f900899..d0a6ca96495 100644 --- a/CIME/SystemTests/seq.py +++ b/CIME/SystemTests/seq.py @@ -1,7 +1,9 @@ """ sequencing bfb test (10 day seq,conc tests) """ -from CIME.XML.standard_module_setup import * + +import logging + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/sms.py b/CIME/SystemTests/sms.py index 17672b47052..18a42fc529a 100644 --- a/CIME/SystemTests/sms.py +++ b/CIME/SystemTests/sms.py @@ -3,7 +3,8 @@ It does a startup run with restarts off and optionally compares to or generates baselines """ -from CIME.XML.standard_module_setup import * +import logging + from CIME.SystemTests.system_tests_common import SystemTestsCommon logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index 67ed8416671..5eeb3e5aed7 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -2,45 +2,70 @@ Base class for CIME system tests """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_run import EnvRun -from CIME.XML.env_test import EnvTest -from CIME.status import append_testlog -from CIME.utils import ( - get_model, - safe_copy, - get_timestamp, - CIMEError, - expect, - get_current_commit, - SharedArea, - is_comp_standalone, +import calendar +import glob +import gzip +import logging +import math +import os +import re +import time +import traceback +from contextlib import ExitStack +from datetime import datetime, timedelta + +import CIME.build as build +from CIME.baselines.performance import ( + get_latest_cpl_logs, + load_coupler_customization, + perf_compare_memory_baseline, + perf_compare_throughput_baseline, + perf_get_memory_list, + perf_write_baseline, ) -from CIME.test_status import * +from CIME.config import Config from CIME.hist_utils import ( - copy_histfiles, + compare_baseline, compare_test, + copy_histfiles, + generate_baseline, generate_teststatus, - compare_baseline, get_ts_synopsis, - generate_baseline, ) -from CIME.config import Config -from CIME.provenance import save_test_time, get_test_success -from CIME.locked_files import LOCKED_DIR, lock_file, is_locked -from CIME.baselines.performance import ( - get_latest_cpl_logs, - perf_get_memory_list, - perf_compare_memory_baseline, - perf_compare_throughput_baseline, - perf_write_baseline, - load_coupler_customization, +from CIME.locked_files import LOCKED_DIR, is_locked, lock_file +from CIME.provenance import get_test_success, save_test_time +from CIME.status import append_testlog +from CIME.test_status import ( + BASELINE_PHASE, + COMPARE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + MEMLEAK_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SHAREDLIB_BUILD_PHASE, + STARCHIVE_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + THROUGHPUT_PHASE, + TestStatus, ) -import CIME.build as build -from datetime import datetime, timedelta -import glob, gzip, time, traceback, os, math, calendar - -from contextlib import ExitStack +from CIME.utils import ( + CIMEError, + SharedArea, + expect, + get_current_commit, + get_model, + get_timestamp, + is_comp_standalone, + run_cmd, + safe_copy, +) +from CIME.XML.env_run import EnvRun +from CIME.XML.env_test import EnvTest logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_n.py b/CIME/SystemTests/system_tests_compare_n.py index 93724eeef3e..b9d49edfbb6 100644 --- a/CIME/SystemTests/system_tests_compare_n.py +++ b/CIME/SystemTests/system_tests_compare_n.py @@ -39,13 +39,16 @@ Use this to do arbitrary actions immediately after running case one """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config -from CIME.test_status import * - -import shutil, os, glob +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index 61232b34117..9652a21a6fa 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -47,13 +47,16 @@ Use this to do arbitrary actions immediately after running case two """ -from CIME.XML.standard_module_setup import * -from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +import glob +import logging +import os +import shutil + from CIME.case import Case from CIME.config import Config -from CIME.test_status import * - -import shutil, os, glob +from CIME.SystemTests.system_tests_common import SystemTestsCommon, fix_single_exe_case +from CIME.test_status import COMPARE_PHASE, RUN_PHASE, TEST_PEND_STATUS +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/SystemTests/test_utils/user_nl_utils.py b/CIME/SystemTests/test_utils/user_nl_utils.py index 930d683666b..00cfdad37d7 100644 --- a/CIME/SystemTests/test_utils/user_nl_utils.py +++ b/CIME/SystemTests/test_utils/user_nl_utils.py @@ -2,8 +2,8 @@ This module contains functions for working with user_nl files in system tests. """ -import os import glob +import os def append_to_user_nl_files(caseroot, component, contents): diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index be695b7f482..5c16a068752 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -7,23 +7,22 @@ This class inherits from SystemTestsCommon. """ -import os import json import logging - +import os from shutil import copytree +import evv4esm # pylint: disable=import-error +from evv4esm.__main__ import main as evv # pylint: disable=import-error + import CIME.test_status import CIME.utils -from CIME.status import append_testlog -from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.hist_utils import rename_all_hist_files +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.XML.machines import Machines -import evv4esm # pylint: disable=import-error -from evv4esm.__main__ import main as evv # pylint: disable=import-error - evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) diff --git a/CIME/Tools/archive_metadata b/CIME/Tools/archive_metadata index 191050254f9..4054db8612d 100755 --- a/CIME/Tools/archive_metadata +++ b/CIME/Tools/archive_metadata @@ -6,25 +6,26 @@ via a web post and SVN check-in Author: CSEG """ import argparse +import configparser import datetime import filecmp import getpass import glob import gzip -import json import io +import json import os -from os.path import expanduser import re import shutil import ssl import subprocess import sys -from string import Template -import configparser import urllib +from os.path import expanduser +from string import Template from standard_script_setup import * + from CIME.case import Case from CIME.utils import is_last_process_complete diff --git a/CIME/Tools/bld_diff b/CIME/Tools/bld_diff index 27193d61560..ea614562886 100755 --- a/CIME/Tools/bld_diff +++ b/CIME/Tools/bld_diff @@ -5,10 +5,15 @@ Try to calculate and succinctly present the differences between two bld logs for the same component """ +import argparse +import gzip +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd_no_fail -import argparse, sys, os, gzip ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/bless_test_results b/CIME/Tools/bless_test_results index 093ae0d79b8..335f1d6a0d4 100755 --- a/CIME/Tools/bless_test_results +++ b/CIME/Tools/bless_test_results @@ -8,16 +8,14 @@ blessing of diffs. You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import expect -from CIME.XML.machines import Machines from CIME.bless_test_results import bless_test_results - -import argparse -import sys -import os -import logging +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/case.build b/CIME/Tools/case.build index 131ca94564d..d69384de9f7 100755 --- a/CIME/Tools/case.build +++ b/CIME/Tools/case.build @@ -46,8 +46,14 @@ from standard_script_setup import * import CIME.build as build from CIME.case import Case +from CIME.test_status import ( + MODEL_BUILD_PHASE, + SHAREDLIB_BUILD_PHASE, + TEST_FAIL_STATUS, + TestStatus, +) from CIME.utils import find_system_test, get_model -from CIME.test_status import * + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.cmpgen_namelists b/CIME/Tools/case.cmpgen_namelists index c9930e71795..e97936f336b 100755 --- a/CIME/Tools/case.cmpgen_namelists +++ b/CIME/Tools/case.cmpgen_namelists @@ -5,10 +5,12 @@ case.cmpgen_namelists - perform namelist baseline operations (compare, generate, or both) for this case. """ +from argparse import RawTextHelpFormatter + from standard_script_setup import * from CIME.case import Case -from argparse import RawTextHelpFormatter + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.qstatus b/CIME/Tools/case.qstatus index 4902dc511d5..305f7c78c69 100755 --- a/CIME/Tools/case.qstatus +++ b/CIME/Tools/case.qstatus @@ -10,7 +10,6 @@ Typical usage is simply: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/case.setup b/CIME/Tools/case.setup index 75eeadd05b0..0a7f4d6937b 100755 --- a/CIME/Tools/case.setup +++ b/CIME/Tools/case.setup @@ -19,9 +19,10 @@ To rerun after making changes to env_mach_pes.xml or env_mach_specific.xml, run: from standard_script_setup import * from CIME.case import Case -from CIME.test_status import * +from CIME.test_status import SETUP_PHASE, TEST_FAIL_STATUS, TestStatus from CIME.utils import find_system_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case.submit b/CIME/Tools/case.submit index 5de69368100..1f4dee5a81c 100755 --- a/CIME/Tools/case.submit +++ b/CIME/Tools/case.submit @@ -16,10 +16,13 @@ Other examples: """ import configparser + from standard_script_setup import * + from CIME.case import Case from CIME.utils import expect + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/case_diff b/CIME/Tools/case_diff index e642dd8e7ac..167845ed91d 100755 --- a/CIME/Tools/case_diff +++ b/CIME/Tools/case_diff @@ -5,10 +5,14 @@ Try to calculate and succinctly present the differences between two large directory trees. """ +import argparse +import os +import sys + from standard_script_setup import * + from CIME.utils import run_cmd, run_cmd_no_fail -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_case b/CIME/Tools/check_case index 9b061046643..30bf1361019 100755 --- a/CIME/Tools/check_case +++ b/CIME/Tools/check_case @@ -17,13 +17,13 @@ automatically when running case.submit. However, you can run this if you want to perform these checks without actually submitting the case. """ +import argparse + from standard_script_setup import * -from CIME.utils import expect from CIME.case import Case from CIME.locked_files import check_lockedfiles - -import argparse +from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/check_input_data b/CIME/Tools/check_input_data index 179af50c4fd..df11a8b539d 100755 --- a/CIME/Tools/check_input_data +++ b/CIME/Tools/check_input_data @@ -18,10 +18,12 @@ To obtain missing datasets from the input data server(s) use: This script is automatically called by the case control system, when the case is built and submitted. So manual usage of this script is optional. """ +import argparse + from standard_script_setup import * + from CIME.case import Case -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/check_lockedfiles b/CIME/Tools/check_lockedfiles index a959b953570..068f24793f5 100755 --- a/CIME/Tools/check_lockedfiles +++ b/CIME/Tools/check_lockedfiles @@ -4,6 +4,7 @@ This script compares xml files """ from standard_script_setup import * + from CIME.case import Case from CIME.locked_files import check_lockedfiles diff --git a/CIME/Tools/cime_bisect b/CIME/Tools/cime_bisect index e427679f692..0a4969599fd 100755 --- a/CIME/Tools/cime_bisect +++ b/CIME/Tools/cime_bisect @@ -9,10 +9,14 @@ NOTE: this tool will only work for models that use git and, for bisecting CIME, bring in CIME via submodule or clone. """ +import argparse +import os +import re +import sys + from standard_script_setup import * -from CIME.utils import expect, run_cmd_no_fail, run_cmd -import argparse, sys, os, re +from CIME.utils import expect, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/Tools/code_checker b/CIME/Tools/code_checker index 6f7510337ab..b84c5278c22 100755 --- a/CIME/Tools/code_checker +++ b/CIME/Tools/code_checker @@ -5,15 +5,17 @@ Ensure that all CIME python files are free of errors and follow the PEP8 standard. """ -from standard_script_setup import * - -from CIME.code_checker import check_code, expect - -import argparse, sys, os +import argparse +import os +import sys # pylint: disable=import-error from shutil import which +from standard_script_setup import * + +from CIME.code_checker import check_code, expect + logger = logging.getLogger(__name__) ############################################################################### diff --git a/CIME/Tools/compare_namelists b/CIME/Tools/compare_namelists index 5aba49edbc0..9e71be71397 100755 --- a/CIME/Tools/compare_namelists +++ b/CIME/Tools/compare_namelists @@ -5,11 +5,15 @@ Compare namelists. Should be called by an ACME test. Designed to not be sensitive to order or whitespace. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.compare_namelists from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/compare_test_results b/CIME/Tools/compare_test_results index 3b59ccd007f..ce7be67c038 100755 --- a/CIME/Tools/compare_test_results +++ b/CIME/Tools/compare_test_results @@ -21,12 +21,14 @@ You may need to load modules for cprnc to work. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.XML.machines import Machines from CIME.compare_test_results import compare_test_results - -import argparse, sys, os +from CIME.XML.machines import Machines _MACHINE = Machines() diff --git a/CIME/Tools/component_compare_baseline b/CIME/Tools/component_compare_baseline index 2adad3b94ad..8a15a3af8cf 100755 --- a/CIME/Tools/component_compare_baseline +++ b/CIME/Tools/component_compare_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_copy b/CIME/Tools/component_compare_copy index 28005cd6674..3ed0dbecac0 100755 --- a/CIME/Tools/component_compare_copy +++ b/CIME/Tools/component_compare_copy @@ -10,6 +10,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import copy_histfiles + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_compare_test b/CIME/Tools/component_compare_test index ece602e1af4..d4600a4db76 100755 --- a/CIME/Tools/component_compare_test +++ b/CIME/Tools/component_compare_test @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import compare_test + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/component_generate_baseline b/CIME/Tools/component_generate_baseline index ff8f39170f0..565ee8c5771 100755 --- a/CIME/Tools/component_generate_baseline +++ b/CIME/Tools/component_generate_baseline @@ -9,6 +9,7 @@ from standard_script_setup import * from CIME.case import Case from CIME.hist_utils import generate_baseline + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/cs.status b/CIME/Tools/cs.status index 4ab7b7e8ecd..e8e7ccd8cd9 100755 --- a/CIME/Tools/cs.status +++ b/CIME/Tools/cs.status @@ -8,11 +8,15 @@ Typical usage: Returns True if no errors occured (not based on test statuses). """ -from standard_script_setup import * -import argparse, sys, os, logging, glob -from CIME.utils import expect -from CIME.cs_status import cs_status +import argparse +import glob +import os +import sys + + from CIME import test_status +from CIME.cs_status import cs_status +from CIME.utils import expect _PERFORMANCE_PHASES = [test_status.THROUGHPUT_PHASE, test_status.MEMCOMP_PHASE] diff --git a/CIME/Tools/e3sm_check_env b/CIME/Tools/e3sm_check_env index b1756c9cfd3..79f593a7b26 100755 --- a/CIME/Tools/e3sm_check_env +++ b/CIME/Tools/e3sm_check_env @@ -6,10 +6,13 @@ A script to verify that the environment is compliant with E3SM's software requir Be sure to source your env_mach_specific file before running this check. """ +import argparse +import os +import sys + from standard_script_setup import * -from CIME.utils import run_cmd -import sys, os, argparse +from CIME.utils import run_cmd # Here's where we keep the various reports and instructions. LOG = [] diff --git a/CIME/Tools/generate_cylc_workflow.py b/CIME/Tools/generate_cylc_workflow.py index cac503b342b..b7465359228 100755 --- a/CIME/Tools/generate_cylc_workflow.py +++ b/CIME/Tools/generate_cylc_workflow.py @@ -3,20 +3,22 @@ """ Generates a cylc workflow file for the case. See https://cylc.github.io for details about cylc """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import argparse +import re from CIME.case import Case +from CIME.Tools.standard_script_setup import * from CIME.utils import expect, transform_vars -import argparse, re - logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/Tools/getTiming b/CIME/Tools/getTiming index 3f16847385a..7a1352d714e 100755 --- a/CIME/Tools/getTiming +++ b/CIME/Tools/getTiming @@ -3,8 +3,12 @@ Get timing information from run """ +import argparse +import os +import sys + from standard_script_setup import * -import argparse, sys, os + from CIME.case import Case from CIME.get_timing import get_timing diff --git a/CIME/Tools/get_case_env b/CIME/Tools/get_case_env index 7e7db4b393a..9ddd2fe462a 100755 --- a/CIME/Tools/get_case_env +++ b/CIME/Tools/get_case_env @@ -5,14 +5,18 @@ Dump what the CIME environment would be for a case. Only supports E3SM for now. """ +import argparse +import shutil +import tempfile + from standard_script_setup import * -from CIME.XML.machines import Machines -from CIME.test_scheduler import TestScheduler -from CIME.utils import parse_test_name, expect, get_src_root -from CIME.config import Config + import CIME.get_tests +from CIME.config import Config +from CIME.test_scheduler import TestScheduler +from CIME.utils import expect, get_src_root +from CIME.XML.machines import Machines -import argparse, tempfile, shutil ############################################################################### def parse_command_line(raw_args, description): diff --git a/CIME/Tools/get_standard_makefile_args b/CIME/Tools/get_standard_makefile_args index 5357c09cc1a..2c002717125 100755 --- a/CIME/Tools/get_standard_makefile_args +++ b/CIME/Tools/get_standard_makefile_args @@ -9,7 +9,6 @@ from standard_script_setup import * from CIME.build import get_standard_makefile_args from CIME.case import Case -from CIME.test_status import * ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index b02a5b69199..b4228524260 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -10,9 +10,10 @@ ensures that the batch system is left in a clean state. from standard_script_setup import * import CIME.wait_for_tests +from CIME.jenkins_generic_job import jenkins_generic_job from CIME.utils import expect from CIME.XML.machines import Machines -from CIME.jenkins_generic_job import jenkins_generic_job + ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/list_e3sm_tests b/CIME/Tools/list_e3sm_tests index 58bb436da76..8bf69d94cc0 100755 --- a/CIME/Tools/list_e3sm_tests +++ b/CIME/Tools/list_e3sm_tests @@ -5,16 +5,14 @@ List e3sm test suites. Can be used to show what's being tested. Can just list tested grids, compsets, etc. """ -from standard_script_setup import * -from CIME import utils -from CIME import get_tests -from CIME.XML.files import Files -from CIME.XML.compsets import Compsets - import argparse import logging +from CIME import get_tests, utils +from CIME.XML.compsets import Compsets +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/Tools/mvsource b/CIME/Tools/mvsource index e02f20c91bf..0b3d042af3e 100755 --- a/CIME/Tools/mvsource +++ b/CIME/Tools/mvsource @@ -2,7 +2,8 @@ # This tool expects two arguments, the caseroot and the new cimeroot # It is intended to fix links and update your case source if the caseroot or src is moved. # -import os, sys +import os +import sys caseroot = sys.argv[1] cimeroot = sys.argv[2] @@ -12,7 +13,8 @@ sys.path.append(_LIBDIR) _LIBDIR = os.path.join(cimeroot, "scripts", "lib") sys.path.append(_LIBDIR) try: - from standard_script_setup import * + pass + from CIME.case import Case from CIME.utils import symlink_force except: diff --git a/CIME/Tools/normalize_cases b/CIME/Tools/normalize_cases index 6fa0e8111dc..2d7d35e6915 100755 --- a/CIME/Tools/normalize_cases +++ b/CIME/Tools/normalize_cases @@ -8,10 +8,15 @@ This is for debugging purposes and meant to assist the user when they want to run case_diff. """ +import argparse +import glob +import os +import sys + from standard_script_setup import * + from CIME.utils import expect, run_cmd_no_fail -import argparse, sys, os, glob ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/pelayout b/CIME/Tools/pelayout index d4c3575359b..c109435c96c 100755 --- a/CIME/Tools/pelayout +++ b/CIME/Tools/pelayout @@ -38,12 +38,13 @@ If you encounter problems with this tool or find it is missing any feature that you need, please open an issue on https://github.com/ESMCI/cime """ +import re +import sys + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string -import sys -import re +from CIME.utils import convert_to_string, expect logger = logging.getLogger(__name__) diff --git a/CIME/Tools/preview_namelists b/CIME/Tools/preview_namelists index 69b89eb5663..0cc3dc988ae 100755 --- a/CIME/Tools/preview_namelists +++ b/CIME/Tools/preview_namelists @@ -15,12 +15,13 @@ Typical usage is simply: ./preview_namelists """ +import argparse + from standard_script_setup import * from CIME.case import Case from CIME.utils import expect -import argparse ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/save_provenance b/CIME/Tools/save_provenance index 7e3506ff50c..ce4498e0e62 100755 --- a/CIME/Tools/save_provenance +++ b/CIME/Tools/save_provenance @@ -4,15 +4,15 @@ This tool provide command-line access to provenance-saving functionality """ +import logging + from standard_script_setup import * from CIME.case import Case from CIME.config import Config +from CIME.get_timing import get_timing from CIME.provenance import * from CIME.utils import get_lids -from CIME.get_timing import get_timing - -import logging logger = logging.getLogger(__name__) diff --git a/CIME/Tools/simple_compare b/CIME/Tools/simple_compare index 59bc328341c..034f3862337 100755 --- a/CIME/Tools/simple_compare +++ b/CIME/Tools/simple_compare @@ -5,11 +5,15 @@ Compare files in a normalized way. Used by create_test for diffing non-namelist files. """ +import argparse +import os +import sys + from standard_script_setup import * + import CIME.simple_compare from CIME.utils import expect -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/standard_script_setup.py b/CIME/Tools/standard_script_setup.py index 2c38a5c4035..f82dcd65ed1 100644 --- a/CIME/Tools/standard_script_setup.py +++ b/CIME/Tools/standard_script_setup.py @@ -1,11 +1,19 @@ """ Encapsulate the importing of python utils and logging setup, things that every script should do. + +This module is DEPRECATED for star imports. New scripts should use explicit +imports instead of `from CIME.Tools.standard_script_setup import *`. + +The exports are maintained for backward compatibility. """ + # pylint: disable=unused-import -import sys, os -import __main__ as main +import argparse # noqa: F401 +import logging # noqa: F401 +import os +import sys def check_minimum_python_version(major, minor, warn_only=False): @@ -48,7 +56,6 @@ def check_minimum_python_version(major, minor, warn_only=False): # Important: Allows external tools to link up with CIME os.environ["CIMEROOT"] = cimeroot -import CIME.utils +import CIME.utils # noqa: E402 CIME.utils.stop_buffering_output() -import argparse, logging diff --git a/CIME/Tools/testreporter.py b/CIME/Tools/testreporter.py index a76c113ac09..7397d1f72c8 100755 --- a/CIME/Tools/testreporter.py +++ b/CIME/Tools/testreporter.py @@ -3,21 +3,22 @@ """ Simple script to populate CESM test database with test results. """ + import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) -from CIME.Tools.standard_script_setup import * +import glob +from CIME.Tools.standard_script_setup import * +from CIME.utils import expect from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_test import EnvTest -from CIME.XML.test_reporter import TestReporter -from CIME.utils import expect from CIME.XML.generic_xml import GenericXML +from CIME.XML.test_reporter import TestReporter -import glob ############################################################################### def parse_command_line(args): diff --git a/CIME/Tools/wait_for_tests b/CIME/Tools/wait_for_tests index 484b393b6f4..19e298d861e 100755 --- a/CIME/Tools/wait_for_tests +++ b/CIME/Tools/wait_for_tests @@ -8,11 +8,13 @@ for the RUN phase specifically and will not terminate if the RUN phase didn't happen. """ -from standard_script_setup import * +import argparse +import os +import sys + import CIME.wait_for_tests -import argparse, sys, os ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlchange b/CIME/Tools/xmlchange index 4101f9b5dba..23b76f95e53 100755 --- a/CIME/Tools/xmlchange +++ b/CIME/Tools/xmlchange @@ -50,18 +50,18 @@ Examples: ./xmlchange JOB_WALLCLOCK_TIME=0:30 """ +import re + from standard_script_setup import * +from CIME.case import Case +from CIME.locked_files import check_lockedfiles +from CIME.status import append_case_status from CIME.utils import ( - expect, - convert_to_type, Timeout, + convert_to_type, + expect, ) -from CIME.status import append_case_status -from CIME.case import Case -from CIME.locked_files import check_lockedfiles - -import re # Set logger logger = logging.getLogger("xmlchange") diff --git a/CIME/Tools/xmlconvertors/config_pes_converter.py b/CIME/Tools/xmlconvertors/config_pes_converter.py index cf57c21fb9b..2c36090199d 100755 --- a/CIME/Tools/xmlconvertors/config_pes_converter.py +++ b/CIME/Tools/xmlconvertors/config_pes_converter.py @@ -8,21 +8,26 @@ CIME2: cime/machines-acme/config_pes.xml CIME5: config/acme/allactive/config_pesall.xml """ -import os, sys + +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import xml.etree.ElementTree as ET +from shutil import which + +import grid_xml_converter + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd -from shutil import which -import xml.etree.ElementTree as ET -import grid_xml_converter LOGGER = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -200,7 +205,9 @@ def set_data(self, xmlnode): class PesTree(grid_xml_converter.DataTree): def __init__(self, xmlfilename): # original xml file has bad comments - import re, StringIO + import re + + import StringIO if os.access(xmlfilename, os.R_OK): with open(xmlfilename, "r") as xmlfile: diff --git a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 index 53b22d0e4a6..42d7a497bff 100755 --- a/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 +++ b/CIME/Tools/xmlconvertors/convert-grid-v1-to-v2 @@ -4,16 +4,19 @@ Convert a grid file from v1 to v2. """ -import argparse, sys, os +import argparse +import os +import sys sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from collections import OrderedDict + from standard_script_setup import * + from CIME.utils import expect from CIME.XML.generic_xml import GenericXML -import xml.etree.ElementTree as ET -from collections import OrderedDict ############################################################################### def parse_command_line(args, description): diff --git a/CIME/Tools/xmlconvertors/grid_xml_converter.py b/CIME/Tools/xmlconvertors/grid_xml_converter.py index 01de9ddcf9f..acd19b94850 100755 --- a/CIME/Tools/xmlconvertors/grid_xml_converter.py +++ b/CIME/Tools/xmlconvertors/grid_xml_converter.py @@ -16,21 +16,24 @@ # CIME5: config/acme/config_grids.xml # -import os, sys +import os +import sys sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) ) +import operator +import xml.etree.ElementTree as ET +from shutil import which + from CIME import utils from CIME.Tools.standard_script_setup import * from CIME.utils import run_cmd_no_fail -from shutil import which -import xml.etree.ElementTree as ET -import operator logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### diff --git a/CIME/Tools/xmlquery b/CIME/Tools/xmlquery index 25ca0a2ea2f..56b6cb4a3d4 100755 --- a/CIME/Tools/xmlquery +++ b/CIME/Tools/xmlquery @@ -97,12 +97,14 @@ There are two usage modes: - The env_mach_specific.xml and env_archive.xml files are not supported by this tool. """ +import re +import sys +import textwrap + from standard_script_setup import * from CIME.case import Case -from CIME.utils import expect, convert_to_string - -import textwrap, sys, re +from CIME.utils import convert_to_string, expect logger = logging.getLogger("xmlquery") unsupported_files = ["env_mach_specific.xml", "env_archive.xml"] diff --git a/CIME/XML/archive.py b/CIME/XML/archive.py index e9d13eae686..8c34780acbb 100644 --- a/CIME/XML/archive.py +++ b/CIME/XML/archive.py @@ -2,11 +2,13 @@ Interface to the archive.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * +import logging +import os +from copy import deepcopy + from CIME.config import Config from CIME.XML.archive_base import ArchiveBase from CIME.XML.files import Files -from copy import deepcopy logger = logging.getLogger(__name__) diff --git a/CIME/XML/archive_base.py b/CIME/XML/archive_base.py index 360424b734d..306cd2b7f71 100644 --- a/CIME/XML/archive_base.py +++ b/CIME/XML/archive_base.py @@ -1,9 +1,13 @@ """ Base class for archive files. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os +import re + +from CIME.utils import convert_to_type, expect from CIME.XML.generic_xml import GenericXML -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/batch.py b/CIME/XML/batch.py index 75d9b1cfb94..fa2213f7c0d 100644 --- a/CIME/XML/batch.py +++ b/CIME/XML/batch.py @@ -4,10 +4,13 @@ The batch_system type="foo" blocks define most things. Machine-specific overrides can be defined by providing a batch_system MACH="mach" block. """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/component.py b/CIME/XML/component.py index abd09c86fd0..6120769ece8 100644 --- a/CIME/XML/component.py +++ b/CIME/XML/component.py @@ -1,11 +1,13 @@ """ Interface to the config_component.xml files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * +import logging +import re + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import get_cime_root logger = logging.getLogger(__name__) diff --git a/CIME/XML/compsets.py b/CIME/XML/compsets.py index 70241ec2e98..415b16a14a2 100644 --- a/CIME/XML/compsets.py +++ b/CIME/XML/compsets.py @@ -2,11 +2,13 @@ Common interface to XML files which follow the compsets format, """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.core.exceptions import CIMEError +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.core.exceptions import CIMEError +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/entry_id.py b/CIME/XML/entry_id.py index 8a7acd92a4b..48927be1df9 100644 --- a/CIME/XML/entry_id.py +++ b/CIME/XML/entry_id.py @@ -3,8 +3,12 @@ this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, convert_to_string, convert_to_type + +import logging +import os +import re + +from CIME.utils import convert_to_string, convert_to_type, expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_archive.py b/CIME/XML/env_archive.py index e3752017aa2..26357452d04 100644 --- a/CIME/XML/env_archive.py +++ b/CIME/XML/env_archive.py @@ -1,12 +1,17 @@ """ Interface to the env_archive.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os + from CIME import utils from CIME.XML.archive_base import ArchiveBase from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) + + # pylint: disable=super-init-not-called class EnvArchive(ArchiveBase, EnvBase): def __init__(self, case_root=None, infile="env_archive.xml", read_only=False): diff --git a/CIME/XML/env_base.py b/CIME/XML/env_base.py index b30d33da988..f0789761333 100644 --- a/CIME/XML/env_base.py +++ b/CIME/XML/env_base.py @@ -1,10 +1,13 @@ """ Base class for env files. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * + +import logging +import os + +from CIME.utils import convert_to_type, expect from CIME.XML.entry_id import EntryID from CIME.XML.headers import Headers -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index fbcc112c56f..f2e5e32088f 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -2,25 +2,30 @@ Interface to the env_batch.xml file. This class inherits from EnvBase """ +import logging +import math import os -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import pathlib +import re +import stat +from collections import OrderedDict +from itertools import zip_longest + from CIME import utils from CIME.utils import ( - transform_vars, - get_cime_root, - convert_to_seconds, + add_flag_to_cmd, convert_to_babylonian_time, - get_cime_config, + convert_to_seconds, + expect, + format_time, get_batch_script_for_job, + get_cime_config, get_logging_options, - format_time, - add_flag_to_cmd, + run_cmd, + run_cmd_no_fail, + transform_vars, ) -from collections import OrderedDict -import stat, re, math -import pathlib -from itertools import zip_longest +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_build.py b/CIME/XML/env_build.py index fe863e414ef..bc72258e1c6 100644 --- a/CIME/XML/env_build.py +++ b/CIME/XML/env_build.py @@ -1,7 +1,9 @@ """ Interface to the env_build.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_case.py b/CIME/XML/env_case.py index 1b4c85d6f88..e0af1f951d6 100644 --- a/CIME/XML/env_case.py +++ b/CIME/XML/env_case.py @@ -1,7 +1,9 @@ """ Interface to the env_case.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * + +import logging +import os from CIME import utils from CIME.XML.env_base import EnvBase diff --git a/CIME/XML/env_mach_pes.py b/CIME/XML/env_mach_pes.py index 86a9cb7d40c..8051ab91dcf 100644 --- a/CIME/XML/env_mach_pes.py +++ b/CIME/XML/env_mach_pes.py @@ -1,10 +1,14 @@ """ Interface to the env_mach_pes.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * + +import logging +import math +import os + from CIME import utils +from CIME.utils import expect from CIME.XML.env_base import EnvBase -import math logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_mach_specific.py b/CIME/XML/env_mach_specific.py index dcd282d614f..d382ca46792 100644 --- a/CIME/XML/env_mach_specific.py +++ b/CIME/XML/env_mach_specific.py @@ -1,14 +1,19 @@ """ Interface to the env_mach_specific.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME import utils -from CIME.utils import transform_vars, get_cime_root -import string, resource, platform +import logging +import os +import platform +import re +import resource +import string from collections import OrderedDict +from CIME import utils +from CIME.utils import expect, run_cmd, run_cmd_no_fail, transform_vars +from CIME.XML.env_base import EnvBase + logger = logging.getLogger(__name__) # Helpful map for converting resource limits to shell commands @@ -32,6 +37,7 @@ "RLIMIT_STACK": ("stacksize", "-s"), } + # Is not of type EntryID but can use functions from EntryID (e.g # get_type) otherwise need to implement own functions and make GenericXML parent class class EnvMachSpecific(EnvBase): diff --git a/CIME/XML/env_postprocessing.py b/CIME/XML/env_postprocessing.py index 90f56f24d64..7346c1c8b7d 100644 --- a/CIME/XML/env_postprocessing.py +++ b/CIME/XML/env_postprocessing.py @@ -1,11 +1,12 @@ """ Interface to the env_postprocessing.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_run.py b/CIME/XML/env_run.py index 2e34c86c8f7..453c101eae7 100644 --- a/CIME/XML/env_run.py +++ b/CIME/XML/env_run.py @@ -1,12 +1,13 @@ """ Interface to the env_run.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging +import os from CIME import utils from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_test.py b/CIME/XML/env_test.py index 9dbe3615eff..cbf331fe76d 100644 --- a/CIME/XML/env_test.py +++ b/CIME/XML/env_test.py @@ -1,10 +1,11 @@ """ Interface to the env_test.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase +import logging + from CIME.utils import convert_to_type +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/env_workflow.py b/CIME/XML/env_workflow.py index 6de05f34e93..fc5d2e4311c 100644 --- a/CIME/XML/env_workflow.py +++ b/CIME/XML/env_workflow.py @@ -2,11 +2,11 @@ Interface to the env_workflow.xml file. This class inherits from EnvBase """ -from CIME.XML.standard_module_setup import * -from CIME.XML.env_base import EnvBase -from CIME.utils import get_cime_root +import logging +import math -import re, math +from CIME.utils import expect +from CIME.XML.env_base import EnvBase logger = logging.getLogger(__name__) diff --git a/CIME/XML/expected_fails_file.py b/CIME/XML/expected_fails_file.py index 82e6ef0a298..f0ba93d0991 100644 --- a/CIME/XML/expected_fails_file.py +++ b/CIME/XML/expected_fails_file.py @@ -42,11 +42,12 @@ """ -from CIME.XML.standard_module_setup import * +import logging +import os from CIME import utils -from CIME.XML.generic_xml import GenericXML from CIME.expected_fails import ExpectedFails +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/files.py b/CIME/XML/files.py index b2612cc8937..8a29283dd4b 100644 --- a/CIME/XML/files.py +++ b/CIME/XML/files.py @@ -2,19 +2,19 @@ Interface to the config_files.xml file. This class inherits from EntryID.py """ -import re +import logging import os -from CIME.XML.standard_module_setup import * +import re -from CIME.XML.entry_id import EntryID from CIME.utils import ( expect, + get_cime_default_driver, get_cime_root, get_config_path, - get_schema_path, get_model, - get_cime_default_driver, + get_schema_path, ) +from CIME.XML.entry_id import EntryID logger = logging.getLogger(__name__) diff --git a/CIME/XML/generic_xml.py b/CIME/XML/generic_xml.py index 6c7ff812a88..e350f108304 100644 --- a/CIME/XML/generic_xml.py +++ b/CIME/XML/generic_xml.py @@ -2,16 +2,19 @@ Common interface to XML files, this is an abstract class and is expected to be used by other XML interface modules and not directly. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy, get_src_root +import getpass +import logging +import os +import re import xml.etree.ElementTree as ET +from collections import namedtuple +from copy import deepcopy # pylint: disable=import-error from shutil import which -import getpass -from copy import deepcopy -from collections import namedtuple + +from CIME.utils import expect, get_cime_root, get_src_root, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) @@ -398,10 +401,12 @@ def get_child(self, name=None, attributes=None, root=None, err_msg=None): ) expect( child, - err_msg - if err_msg - else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Expected one child, found None with name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return child @@ -418,10 +423,12 @@ def get_optional_child(self, name=None, attributes=None, root=None, err_msg=None expect( len(children) <= 1, - err_msg - if err_msg - else "Multiple matches for name '{}' and attribs '{}' in file {}".format( - name, attributes, self.filename + ( + err_msg + if err_msg + else "Multiple matches for name '{}' and attribs '{}' in file {}".format( + name, attributes, self.filename + ) ), ) return children[0] if children else None diff --git a/CIME/XML/grids.py b/CIME/XML/grids.py index 60bc1ff3d02..fbde52f6dc2 100644 --- a/CIME/XML/grids.py +++ b/CIME/XML/grids.py @@ -3,8 +3,12 @@ This is not an abstract class - but inherits from the abstact class GenericXML """ +import logging +import os +import re from collections import OrderedDict -from CIME.XML.standard_module_setup import * + +from CIME.utils import expect from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML diff --git a/CIME/XML/headers.py b/CIME/XML/headers.py index 5937a1d03cb..164ba7bccb2 100644 --- a/CIME/XML/headers.py +++ b/CIME/XML/headers.py @@ -1,10 +1,11 @@ """ Interface to the config_headers.xml file. This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/inputdata.py b/CIME/XML/inputdata.py index 18b71dca4dd..46459ccfff6 100644 --- a/CIME/XML/inputdata.py +++ b/CIME/XML/inputdata.py @@ -1,10 +1,11 @@ """ Interface to the config_inputdata.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML + +import logging + from CIME.XML.files import Files -from CIME.utils import expect +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/machines.py b/CIME/XML/machines.py index eabd6a8f9ef..467c83ead91 100644 --- a/CIME/XML/machines.py +++ b/CIME/XML/machines.py @@ -2,18 +2,18 @@ Interface to the config_machines.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files -from CIME.core.exceptions import CIMEError -from CIME.utils import expect, convert_to_unknown_type, get_cime_config - -import re import logging +import os +import re import socket from functools import partial from pathlib import Path +from CIME.core.exceptions import CIMEError +from CIME.utils import convert_to_unknown_type, expect, get_cime_config +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML + logger = logging.getLogger(__name__) diff --git a/CIME/XML/namelist_definition.py b/CIME/XML/namelist_definition.py index 38a184d1019..ebb35c09582 100644 --- a/CIME/XML/namelist_definition.py +++ b/CIME/XML/namelist_definition.py @@ -10,19 +10,21 @@ # Disable warnings due to using `standard_module_setup` # pylint:disable=wildcard-import,unused-wildcard-import -import re import collections +import logging +import os +import re +from CIME.core.exceptions import CIMEError from CIME.namelist import ( - fortran_namelist_base_value, - is_valid_fortran_namelist_literal, + Namelist, character_literal_to_string, expand_literal_list, - Namelist, + fortran_namelist_base_value, get_fortran_name_only, + is_valid_fortran_namelist_literal, ) -from CIME.core.exceptions import CIMEError -from CIME.XML.standard_module_setup import * +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files diff --git a/CIME/XML/pes.py b/CIME/XML/pes.py index 666382c0a38..e45af99c939 100644 --- a/CIME/XML/pes.py +++ b/CIME/XML/pes.py @@ -2,10 +2,12 @@ Interface to the config_pes.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import re + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/pio.py b/CIME/XML/pio.py index 54af6112bf0..42953263a01 100644 --- a/CIME/XML/pio.py +++ b/CIME/XML/pio.py @@ -1,12 +1,13 @@ """ Class for config_pio files . This class inherits from EntryID.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.entry_id import EntryID -from CIME.XML.files import Files +import logging from collections import OrderedDict +from CIME.XML.entry_id import EntryID +from CIME.XML.files import Files + logger = logging.getLogger(__name__) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index 3287d145142..22144455aaa 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -2,10 +2,12 @@ Interface to the config_postprocessing.xml file. This class inherits from EntryID """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.entry_id import EntryID from CIME.XML.files import Files -from CIME.utils import expect logger = logging.getLogger(__name__) diff --git a/CIME/XML/standard_module_setup.py b/CIME/XML/standard_module_setup.py index 1c934407da5..7957ee191db 100644 --- a/CIME/XML/standard_module_setup.py +++ b/CIME/XML/standard_module_setup.py @@ -1,8 +1,20 @@ # pragma pylint: disable=unused-import +""" +Standard imports for CIME XML modules. -import logging, os, sys, re +This module is DEPRECATED. New code should use explicit imports instead of +`from CIME.XML.standard_module_setup import *`. + +The exports are maintained for backward compatibility with external code +(E3SM, CESM, NorESM cime_config directories). +""" + +import logging # noqa: F401 +import os # noqa: F401 +import re # noqa: F401 +import sys # noqa: F401 LIB_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(LIB_DIR) -from CIME.utils import expect, run_cmd, run_cmd_no_fail, get_cime_root +from CIME.utils import expect, get_cime_root, run_cmd, run_cmd_no_fail # noqa: F401 diff --git a/CIME/XML/stream.py b/CIME/XML/stream.py index 95dfa30ef7d..1418a09da09 100644 --- a/CIME/XML/stream.py +++ b/CIME/XML/stream.py @@ -3,10 +3,13 @@ stream files predate cime and so do not conform to entry id format """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files + +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/test_reporter.py b/CIME/XML/test_reporter.py index 93f117a8a46..17003715d3f 100644 --- a/CIME/XML/test_reporter.py +++ b/CIME/XML/test_reporter.py @@ -2,12 +2,15 @@ Interface to the testreporter xml. This class inherits from GenericXML.py """ + +import os +import ssl + # pylint: disable=import-error import urllib.parse import urllib.request -from CIME.XML.standard_module_setup import * + from CIME.XML.generic_xml import GenericXML -import ssl # pylint: disable=protected-access ssl._create_default_https_context = ssl._create_unverified_context diff --git a/CIME/XML/testlist.py b/CIME/XML/testlist.py index 6dbe79b8f98..dbe75db6332 100644 --- a/CIME/XML/testlist.py +++ b/CIME/XML/testlist.py @@ -35,10 +35,12 @@ - workflow: adds a workflow to the test """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML +import logging + +from CIME.utils import expect from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/tests.py b/CIME/XML/tests.py index 92feaca91c8..bcef473d5e5 100644 --- a/CIME/XML/tests.py +++ b/CIME/XML/tests.py @@ -2,14 +2,15 @@ Interface to the config_tests.xml file. This class inherits from GenericEntry """ -from CIME.XML.standard_module_setup import * +import logging +import os -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files from CIME.core.exceptions import CIMEError -from CIME.utils import find_system_test -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.utils import find_system_test +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/testspec.py b/CIME/XML/testspec.py index 9b4e7c37724..5686a800f37 100644 --- a/CIME/XML/testspec.py +++ b/CIME/XML/testspec.py @@ -1,8 +1,11 @@ """ Interface to the testspec.xml file. This class inherits from generic_xml.py """ -from CIME.XML.standard_module_setup import * +import logging +import os + +from CIME.utils import expect from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/XML/workflow.py b/CIME/XML/workflow.py index cdb2d27b403..113ab7b4965 100644 --- a/CIME/XML/workflow.py +++ b/CIME/XML/workflow.py @@ -2,10 +2,12 @@ Interface to the config_workflow.xml file. This class inherits from GenericXML.py """ -from CIME.XML.standard_module_setup import * -from CIME.XML.generic_xml import GenericXML -from CIME.XML.files import Files +import logging +import os + from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/aprun.py b/CIME/aprun.py index ca767c88db8..65b391dd071 100755 --- a/CIME/aprun.py +++ b/CIME/aprun.py @@ -3,12 +3,12 @@ code to compute and assemble aprun commands. """ -from CIME.XML.standard_module_setup import * - +import logging import math logger = logging.getLogger(__name__) + ############################################################################### def _get_aprun_cmd_for_case_impl( ntasks, diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index 67c19dbd43f..a46d4304265 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -1,10 +1,11 @@ -import os import glob -import re import gzip import logging +import os +import re + from CIME.config import Config -from CIME.utils import expect, get_src_root, get_current_commit, get_timestamp +from CIME.utils import expect, get_current_commit, get_src_root, get_timestamp logger = logging.getLogger(__name__) diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index 3028cfa4390..f4066969e84 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -1,24 +1,40 @@ -import CIME.compare_namelists, CIME.simple_compare -from CIME.test_scheduler import NAMELIST_PHASE -from CIME.utils import ( - run_cmd, - get_scripts_root, - EnvironmentContext, - parse_test_name, - match_any, - CIMEError, -) -from CIME.config import Config -from CIME.test_status import * -from CIME.hist_utils import generate_baseline, compare_baseline, NO_ORIGINAL -from CIME.case import Case -from CIME.test_utils import get_test_status_files +import logging +import os +import re +import time + +import CIME.compare_namelists +import CIME.simple_compare from CIME.baselines.performance import ( - perf_compare_throughput_baseline, perf_compare_memory_baseline, + perf_compare_throughput_baseline, perf_write_baseline, ) -import os, time +from CIME.case import Case +from CIME.config import Config +from CIME.hist_utils import NO_ORIGINAL, compare_baseline, generate_baseline +from CIME.test_status import ( + ALL_PHASES, + BASELINE_PHASE, + GENERATE_PHASE, + MEMCOMP_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + THROUGHPUT_PHASE, + TestStatus, +) +from CIME.test_utils import get_test_status_files +from CIME.utils import ( + CIMEError, + EnvironmentContext, + expect, + get_scripts_root, + match_any, + parse_test_name, + run_cmd, +) logger = logging.getLogger(__name__) diff --git a/CIME/build.py b/CIME/build.py index af1709bb59c..66208d7af22 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -2,26 +2,35 @@ functions for building CIME models """ -import glob, shutil, time, threading, subprocess +import glob +import logging +import os +import re +import shutil +import subprocess +import threading +import time from pathlib import Path -from CIME.XML.standard_module_setup import * + +from CIME.config import Config +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file from CIME.status import run_and_log_case_status from CIME.utils import ( - get_model, analyze_build_log, - stringify_bool, - get_timestamp, - run_sub_or_cmd, - run_cmd, + expect, get_batch_script_for_job, - gzip_existing_file, - safe_copy, - is_python_executable, get_logging_options, + get_model, + get_timestamp, + gzip_existing_file, import_from_file, + is_python_executable, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, + stringify_bool, ) -from CIME.config import Config -from CIME.locked_files import lock_file, unlock_file, check_lockedfiles from CIME.XML.files import Files logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.cprnc b/CIME/build_scripts/buildlib.cprnc index 51426de27c6..187fb5c53f1 100755 --- a/CIME/build_scripts/buildlib.cprnc +++ b/CIME/build_scripts/buildlib.cprnc @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging, CIMEError -from CIME.case import Case from CIME.build import get_standard_cmake_args +from CIME.case import Case +from CIME.utils import CIMEError, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.gptl b/CIME/build_scripts/buildlib.gptl index 774e5659396..786943dbc53 100755 --- a/CIME/build_scripts/buildlib.gptl +++ b/CIME/build_scripts/buildlib.gptl @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -from standard_script_setup import * + from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.internal_components b/CIME/build_scripts/buildlib.internal_components index f7f597d776e..5d79e5234f5 100755 --- a/CIME/build_scripts/buildlib.internal_components +++ b/CIME/build_scripts/buildlib.internal_components @@ -5,14 +5,15 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: raise ValueError("ERROR: CIMEROOT not defined in buildlib.internal_components.") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/build_scripts/buildlib.kokkos b/CIME/build_scripts/buildlib.kokkos index 6ecceb6ae48..5ed918d6332 100755 --- a/CIME/build_scripts/buildlib.kokkos +++ b/CIME/build_scripts/buildlib.kokkos @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -import os, sys, argparse, logging +import argparse +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd_no_fail, run_cmd -from CIME.case import Case from CIME.build import get_standard_makefile_args +from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, run_cmd, run_cmd_no_fail logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mct b/CIME/build_scripts/buildlib.mct index 446020839e4..884572f7a51 100755 --- a/CIME/build_scripts/buildlib.mct +++ b/CIME/build_scripts/buildlib.mct @@ -1,16 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys _CIMEROOT = os.getenv("CIMEROOT") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * -from CIME.config import Config +import glob + + from CIME import utils -from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect -from CIME.case import Case from CIME.build import get_standard_makefile_args -import glob +from CIME.case import Case +from CIME.config import Config +from CIME.utils import copyifnewer, expect, run_bld_cmd_ensure_logging logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.mpi-serial b/CIME/build_scripts/buildlib.mpi-serial index f13d949ae45..b851a1139dc 100755 --- a/CIME/build_scripts/buildlib.mpi-serial +++ b/CIME/build_scripts/buildlib.mpi-serial @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import os, sys, logging, argparse +import argparse +import glob +import logging +import os +import sys + -from standard_script_setup import * from CIME import utils +from CIME.build import get_standard_makefile_args +from CIME.case import Case from CIME.config import Config from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging -from CIME.case import Case -from CIME.build import get_standard_makefile_args -import glob logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib.pio b/CIME/build_scripts/buildlib.pio index b04367b7b19..2e7a2935446 100755 --- a/CIME/build_scripts/buildlib.pio +++ b/CIME/build_scripts/buildlib.pio @@ -1,15 +1,20 @@ #!/usr/bin/env python3 -import sys, os, logging, argparse +import argparse +import logging +import os +import sys cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -import glob, re -from standard_script_setup import * +import glob +import re + + from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/build_scripts/buildlib_cmake.internal_components b/CIME/build_scripts/buildlib_cmake.internal_components index 4c939041e62..b8a56583973 100755 --- a/CIME/build_scripts/buildlib_cmake.internal_components +++ b/CIME/build_scripts/buildlib_cmake.internal_components @@ -5,7 +5,8 @@ build cime component model library. This buildlib script is used by all cime i components. """ -import sys, os +import os +import sys _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT == None: @@ -14,7 +15,7 @@ if _CIMEROOT == None: ) sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) -from standard_script_setup import * + from CIME.buildlib import build_cime_component_lib, parse_input from CIME.case import Case diff --git a/CIME/buildlib.py b/CIME/buildlib.py index b1152f8923a..a487734423c 100644 --- a/CIME/buildlib.py +++ b/CIME/buildlib.py @@ -2,21 +2,24 @@ common utilities for buildlib """ -from CIME.XML.standard_module_setup import * +import argparse +import logging +import os + +from CIME.build import get_standard_makefile_args from CIME.case import Case +from CIME.config import Config from CIME.utils import ( parse_args_and_handle_standard_logging_options, - setup_standard_logging_options, + run_cmd, safe_copy, + setup_standard_logging_options, ) -from CIME.config import Config -from CIME.build import get_standard_makefile_args from CIME.XML.files import Files -import sys, os, argparse - logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/buildnml.py b/CIME/buildnml.py index 69cbc5f7dca..d16e05f1ab6 100644 --- a/CIME/buildnml.py +++ b/CIME/buildnml.py @@ -4,17 +4,22 @@ These are used by components///cime_config/buildnml """ -from CIME.XML.standard_module_setup import * +import argparse +import glob +import logging +import os +import re + from CIME.utils import ( expect, parse_args_and_handle_standard_logging_options, + safe_copy, setup_standard_logging_options, ) -from CIME.utils import safe_copy -import sys, os, argparse, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_input(argv): ############################################################################### diff --git a/CIME/case/case.py b/CIME/case/case.py index bd6db1e503c..b7f549b7ac9 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -5,46 +5,64 @@ All interaction with and between the module files in XML/ takes place through the Case module. """ -from copy import deepcopy + +import getpass +import glob +import hashlib +import logging +import math +import os +import re +import shutil +import socket import sys -import glob, os, shutil, math, time, hashlib, socket, getpass -from CIME.XML.standard_module_setup import * +import time +from copy import deepcopy # pylint: disable=import-error,redefined-builtin from CIME import utils +from CIME.aprun import get_aprun_cmd_for_case from CIME.config import Config -from CIME.status import append_status -from CIME.utils import expect, get_cime_root -from CIME.utils import convert_to_type, get_model, set_model -from CIME.utils import get_project, get_charge_account, check_name -from CIME.utils import get_current_commit, safe_copy, get_cime_default_driver from CIME.gitinterface import GitInterface from CIME.locked_files import LOCKED_DIR, lock_file -from CIME.XML.machines import Machines -from CIME.XML.pes import Pes -from CIME.XML.files import Files -from CIME.XML.testlist import Testlist +from CIME.status import append_status +from CIME.user_mod_support import apply_user_mods +from CIME.utils import ( + check_name, + convert_to_type, + expect, + get_charge_account, + get_cime_default_driver, + get_cime_root, + get_current_commit, + get_model, + get_project, + safe_copy, + set_model, +) +from CIME.XML.archive import Archive +from CIME.XML.batch import Batch from CIME.XML.component import Component from CIME.XML.compsets import Compsets -from CIME.XML.grids import Grids -from CIME.XML.batch import Batch -from CIME.XML.workflow import Workflow -from CIME.XML.postprocessing import Postprocessing -from CIME.XML.pio import PIO -from CIME.XML.archive import Archive -from CIME.XML.env_test import EnvTest -from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_archive import EnvArchive +from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_build import EnvBuild from CIME.XML.env_case import EnvCase from CIME.XML.env_mach_pes import EnvMachPes -from CIME.XML.env_build import EnvBuild +from CIME.XML.env_mach_specific import EnvMachSpecific +from CIME.XML.env_postprocessing import EnvPostprocessing from CIME.XML.env_run import EnvRun -from CIME.XML.env_archive import EnvArchive -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_test import EnvTest from CIME.XML.env_workflow import EnvWorkflow -from CIME.XML.env_postprocessing import EnvPostprocessing +from CIME.XML.files import Files from CIME.XML.generic_xml import GenericXML -from CIME.user_mod_support import apply_user_mods -from CIME.aprun import get_aprun_cmd_for_case +from CIME.XML.grids import Grids +from CIME.XML.machines import Machines +from CIME.XML.pes import Pes +from CIME.XML.pio import PIO +from CIME.XML.postprocessing import Postprocessing +from CIME.XML.testlist import Testlist +from CIME.XML.workflow import Workflow logger = logging.getLogger(__name__) @@ -81,25 +99,25 @@ class Case(object): """ - from CIME.case.case_setup import case_setup, _create_case_repo - from CIME.case.case_clone import create_clone, _copy_user_modified_to_clone - from CIME.case.case_test import case_test - from CIME.case.case_submit import check_DA_settings, check_case, submit + from CIME.case.case_clone import _copy_user_modified_to_clone, create_clone + from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists + from CIME.case.case_run import case_run + from CIME.case.case_setup import _create_case_repo, case_setup from CIME.case.case_st_archive import ( + archive_last_restarts, case_st_archive, restore_from_archive, - archive_last_restarts, - test_st_archive, test_env_archive, + test_st_archive, ) - from CIME.case.case_run import case_run - from CIME.case.case_cmpgen_namelists import case_cmpgen_namelists - from CIME.case.preview_namelists import create_dirs, create_namelists + from CIME.case.case_submit import check_case, check_DA_settings, submit + from CIME.case.case_test import case_test from CIME.case.check_input_data import ( check_all_input_data, - stage_refcase, check_input_data, + stage_refcase, ) + from CIME.case.preview_namelists import create_dirs, create_namelists def __init__(self, case_root=None, read_only=True, record=False, non_local=False): if case_root is None: diff --git a/CIME/case/case_clone.py b/CIME/case/case_clone.py index 8ed6ae89948..6f0b070076a 100644 --- a/CIME/case/case_clone.py +++ b/CIME/case/case_clone.py @@ -1,12 +1,16 @@ """ create_clone is a member of the Case class from file case.py """ -import os, glob, shutil -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, check_name, safe_copy, get_model -from CIME.simple_compare import compare_files + +import glob +import logging +import os +import shutil + from CIME.locked_files import lock_file +from CIME.simple_compare import compare_files from CIME.user_mod_support import apply_user_mods +from CIME.utils import check_name, expect, get_cime_root, get_model, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/case/case_cmpgen_namelists.py b/CIME/case/case_cmpgen_namelists.py index 6a2579ada35..a4b016c3998 100644 --- a/CIME/case/case_cmpgen_namelists.py +++ b/CIME/case/case_cmpgen_namelists.py @@ -3,15 +3,25 @@ case_cmpgen_namelists is a member of Class case from file case.py """ -from CIME.XML.standard_module_setup import * - -from CIME.compare_namelists import is_namelist_file, compare_namelist_files +import glob +import logging +import os +import shutil +import stat +import sys +import traceback + +from CIME.compare_namelists import compare_namelist_files, is_namelist_file from CIME.simple_compare import compare_files, compare_runconfigfiles -from CIME.utils import safe_copy, SharedArea from CIME.status import append_status -from CIME.test_status import * - -import os, shutil, traceback, stat, glob +from CIME.test_status import ( + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) +from CIME.utils import SharedArea, expect, safe_copy logger = logging.getLogger(__name__) @@ -41,9 +51,11 @@ def _do_full_nl_comp(case, test, compare_name, baseline_root=None): comments = "NLCOMP\n" for item in all_items_to_compare: baseline_counterpart = os.path.join( - baseline_casedocs - if os.path.dirname(item).endswith("CaseDocs") - else baseline_dir, + ( + baseline_casedocs + if os.path.dirname(item).endswith("CaseDocs") + else baseline_dir + ), os.path.basename(item), ) if not os.path.exists(baseline_counterpart): diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index 5c44c693b14..01b69c6ce29 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -2,17 +2,30 @@ case_run is a member of Class Case '""" -from CIME.XML.standard_module_setup import * +import glob +import logging +import os +import re +import shutil +import time + from CIME.config import Config -from CIME.utils import gzip_existing_file, new_lid from CIME.core.exceptions import CIMEError -from CIME.utils import run_sub_or_cmd, safe_copy, model_log -from CIME.utils import batch_jobid, is_comp_standalone -from CIME.status import append_status, run_and_log_case_status from CIME.get_timing import get_timing from CIME.locked_files import check_lockedfiles - -import shutil, time, sys, os, glob +from CIME.status import append_status, run_and_log_case_status +from CIME.utils import ( + batch_jobid, + expect, + gzip_existing_file, + is_comp_standalone, + model_log, + new_lid, + run_cmd, + run_cmd_no_fail, + run_sub_or_cmd, + safe_copy, +) TERMINATION_TEXT = ("HAS ENDED", "END OF MODEL RUN", "SUCCESSFUL TERMINATION") diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 38d6a4b0336..ece6e60466f 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -3,31 +3,38 @@ case_setup is a member of class Case from file case.py """ +import errno +import logging import os +import shutil -from CIME.XML.standard_module_setup import * -from CIME.config import Config -from CIME.XML.machines import Machines from CIME.BuildTools.configure import ( - generate_env_mach_specific, copy_depends_files, + generate_env_mach_specific, +) +from CIME.config import Config +from CIME.gitinterface import GitInterface +from CIME.locked_files import check_lockedfiles, lock_file, unlock_file +from CIME.status import append_case_status, run_and_log_case_status +from CIME.test_status import ( + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TestStatus, ) from CIME.utils import ( - get_batch_script_for_job, - safe_copy, + CIMEError, + batch_jobid, + copy_local_macros_to_dir, + expect, file_contains_python_function, + get_batch_script_for_job, import_from_file, - copy_local_macros_to_dir, - batch_jobid, run_cmd_no_fail, - CIMEError, + safe_copy, ) -from CIME.status import run_and_log_case_status, append_case_status -from CIME.test_status import * -from CIME.locked_files import unlock_file, lock_file, check_lockedfiles -from CIME.gitinterface import GitInterface - -import errno, shutil +from CIME.XML.machines import Machines logger = logging.getLogger(__name__) @@ -432,9 +439,11 @@ def _case_setup_impl( if ngpus_per_node >= 0: case.set_value( "NGPUS_PER_NODE", - max(1, ngpus_per_node) - if ngpus_per_node <= max_gpus_per_node - else max_gpus_per_node, + ( + max(1, ngpus_per_node) + if ngpus_per_node <= max_gpus_per_node + else max_gpus_per_node + ), ) elif gpu_offload: raise CIMEError( diff --git a/CIME/case/case_st_archive.py b/CIME/case/case_st_archive.py index 932987d1a9a..e8c12692481 100644 --- a/CIME/case/case_st_archive.py +++ b/CIME/case/case_st_archive.py @@ -4,21 +4,26 @@ are members of class Case from file case.py """ -import shutil, glob, re, os +import glob +import logging +import os +import re +import shutil +from os.path import isdir, join -from CIME.XML.standard_module_setup import * +from CIME.date import get_file_date +from CIME.status import run_and_log_case_status from CIME.utils import ( + batch_jobid, + expect, + find_files, ls_sorted_by_mtime, - symlink_force, + run_cmd, safe_copy, - find_files, - batch_jobid, + symlink_force, ) -from CIME.status import run_and_log_case_status -from CIME.date import get_file_date from CIME.XML.archive import Archive from CIME.XML.files import Files -from os.path import isdir, join logger = logging.getLogger(__name__) diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 6e2e7bc6278..2be69e4141b 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -8,17 +8,24 @@ """ import configparser -from CIME.XML.standard_module_setup import * +import logging +import os + from CIME.core.exceptions import CIMEError -from CIME.utils import expect, get_time_in_seconds -from CIME.status import run_and_log_case_status from CIME.locked_files import ( - unlock_file, - lock_file, check_lockedfile, check_lockedfiles, + lock_file, + unlock_file, ) -from CIME.test_status import * +from CIME.status import run_and_log_case_status +from CIME.test_status import ( + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) +from CIME.utils import expect, get_time_in_seconds logger = logging.getLogger(__name__) diff --git a/CIME/case/case_test.py b/CIME/case/case_test.py index c607dd93786..554526171dc 100644 --- a/CIME/case/case_test.py +++ b/CIME/case/case_test.py @@ -3,12 +3,14 @@ case_test is a member of class Case from case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, find_system_test, find_proc_id -from CIME.SystemTests.system_tests_common import * -from CIME.status import append_testlog +import logging +import os +import signal +import sys -import sys, signal +from CIME.status import append_testlog +from CIME.SystemTests.system_tests_common import * +from CIME.utils import expect, find_proc_id, find_system_test def _iter_signal_names(): diff --git a/CIME/case/check_input_data.py b/CIME/case/check_input_data.py index cc67e1c6aca..f85f9cb64da 100644 --- a/CIME/case/check_input_data.py +++ b/CIME/case/check_input_data.py @@ -1,12 +1,16 @@ """ API for checking input for testcase """ -from CIME.XML.standard_module_setup import * -from CIME.utils import SharedArea, find_files, safe_copy, expect -from CIME.XML.inputdata import Inputdata -import CIME.Servers -import glob, hashlib, shutil +import glob +import hashlib +import logging +import os +import shutil + +import CIME.Servers +from CIME.utils import SharedArea, expect, find_files, safe_copy +from CIME.XML.inputdata import Inputdata logger = logging.getLogger(__name__) # The inputdata_checksum.dat file will be read into this hash if it's available diff --git a/CIME/case/preview_namelists.py b/CIME/case/preview_namelists.py index 95d276e039a..e9f5848b5a6 100644 --- a/CIME/case/preview_namelists.py +++ b/CIME/case/preview_namelists.py @@ -3,9 +3,12 @@ create_dirs and create_namelists are members of Class case from file case.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import import_and_run_sub_or_cmd, safe_copy -import time, glob +import glob +import logging +import os +import time + +from CIME.utils import expect, import_and_run_sub_or_cmd, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/code_checker.py b/CIME/code_checker.py index 08a5a021b0f..52fbdd55886 100644 --- a/CIME/code_checker.py +++ b/CIME/code_checker.py @@ -2,24 +2,21 @@ Libraries for checking python code with pylint """ -import os import json +import logging +import os from shutil import which -from CIME.XML.standard_module_setup import * - from CIME.utils import ( - run_cmd, - run_cmd_no_fail, expect, + get_cime_default_driver, get_cime_root, get_src_root, is_python_executable, - get_cime_default_driver, + run_cmd, + run_cmd_no_fail, ) -from multiprocessing.dummy import Pool as ThreadPool - logger = logging.getLogger(__name__) diff --git a/CIME/compare_namelists.py b/CIME/compare_namelists.py index 6e98a7df9a5..7a830ccc18e 100644 --- a/CIME/compare_namelists.py +++ b/CIME/compare_namelists.py @@ -1,4 +1,7 @@ -import os, re, logging +import logging +import os +import re + from CIME.core.exceptions import CIMEError from CIME.utils import expect diff --git a/CIME/compare_test_results.py b/CIME/compare_test_results.py index 35274cc74ba..dce0267d177 100644 --- a/CIME/compare_test_results.py +++ b/CIME/compare_test_results.py @@ -1,12 +1,22 @@ -import CIME.compare_namelists, CIME.simple_compare -from CIME.status import append_status -from CIME.utils import EnvironmentContext, parse_test_name -from CIME.test_status import * -from CIME.hist_utils import compare_baseline, get_ts_synopsis +import logging +import os + +import CIME.compare_namelists +import CIME.simple_compare from CIME.case import Case +from CIME.hist_utils import compare_baseline, get_ts_synopsis +from CIME.status import append_status +from CIME.test_status import ( + BASELINE_PHASE, + NAMELIST_PHASE, + RUN_PHASE, + SETUP_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TestStatus, +) from CIME.test_utils import get_test_status_files - -import os, logging +from CIME.utils import EnvironmentContext, parse_test_name def append_status_cprnc_log(msg, logfile_name, test_dir): diff --git a/CIME/config.py b/CIME/config.py index f63a4bea78d..de34232fc4d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -1,10 +1,10 @@ -import os -import re -import sys -import logging import importlib.machinery import importlib.util import inspect +import logging +import os +import re +import sys from pathlib import Path from CIME import utils @@ -185,7 +185,7 @@ def print_method_rst(self): if x[1].__class__ != Config and x[0] not in ignore ] - for (name, sig, doc) in child_methods: + for name, sig, doc in child_methods: if doc is None: continue print(".. code-block::\n") diff --git a/CIME/cs_status.py b/CIME/cs_status.py index 6a65ca4da71..2ab3de40979 100644 --- a/CIME/cs_status.py +++ b/CIME/cs_status.py @@ -4,13 +4,15 @@ """ from __future__ import print_function -from CIME.XML.standard_module_setup import * -from CIME.XML.expected_fails_file import ExpectedFailsFile -from CIME.test_status import TestStatus, SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS + import os import sys from collections import defaultdict +from CIME.test_status import SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS, TestStatus +from CIME.utils import expect +from CIME.XML.expected_fails_file import ExpectedFailsFile + def cs_status( test_paths, diff --git a/CIME/cs_status_creator.py b/CIME/cs_status_creator.py index 7a0fd40361a..49b456cb788 100644 --- a/CIME/cs_status_creator.py +++ b/CIME/cs_status_creator.py @@ -2,11 +2,11 @@ Creates a test suite-specific cs.status file from a template """ -from CIME.XML.standard_module_setup import * -import CIME.utils import os import stat +import CIME.utils + def create_cs_status(test_root, test_id, extra_args="", filename=None): """Create a test suite-specific cs.status file from the template diff --git a/CIME/date.py b/CIME/date.py index 0d12b2ceede..0e2179b9007 100644 --- a/CIME/date.py +++ b/CIME/date.py @@ -1,7 +1,9 @@ +import logging import re -from CIME.XML.standard_module_setup import * logger = logging.getLogger(__name__) + + ############################################################################### def get_file_date(filename): ############################################################################### diff --git a/CIME/expected_fails.py b/CIME/expected_fails.py index f8e5339702b..5bc457fcba8 100644 --- a/CIME/expected_fails.py +++ b/CIME/expected_fails.py @@ -2,7 +2,7 @@ Contains the definition of a class to hold information on expected failures for a single test """ -from CIME.XML.standard_module_setup import * +from CIME.utils import expect EXPECTED_FAILURE_COMMENT = "(EXPECTED FAILURE)" UNEXPECTED_FAILURE_COMMENT_START = "(UNEXPECTED" # There will be some additional text after this, before the end parentheses diff --git a/CIME/get_tests.py b/CIME/get_tests.py index a4d4abb3410..72c673e850c 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -1,7 +1,9 @@ +import os +import sys + import CIME.utils -from CIME.utils import expect, convert_to_seconds, parse_test_name, get_cime_root +from CIME.utils import convert_to_seconds, expect, get_cime_root, parse_test_name from CIME.XML.machines import Machines -import sys, os # Expect that, if a model wants to use python-based test lists, they will have a file # $model/cime_config/tests.py , containing a test dictionary called _TESTS. Currently, @@ -143,6 +145,7 @@ _ALL_TESTS.update(_CIME_TESTS) + ############################################################################### def _get_key_data(raw_dict, key, the_type): ############################################################################### diff --git a/CIME/get_timing.py b/CIME/get_timing.py index 92ac32509de..ab1127e4518 100644 --- a/CIME/get_timing.py +++ b/CIME/get_timing.py @@ -5,11 +5,13 @@ information from a run. """ -from CIME.XML.standard_module_setup import * -from CIME.utils import safe_copy -from CIME.status import append_case_status +import datetime +import logging +import os +import re -import datetime, re +from CIME.status import append_case_status +from CIME.utils import expect, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/gitinterface.py b/CIME/gitinterface.py index 790f1fc70f2..5c5b4d2423a 100644 --- a/CIME/gitinterface.py +++ b/CIME/gitinterface.py @@ -1,7 +1,10 @@ -import sys, shutil, re -from CIME.utils import run_cmd_no_fail +import re +import shutil +import sys from pathlib import Path +from CIME.utils import run_cmd_no_fail + class GitInterface: def __init__(self, repo_path, logger, branch=None): diff --git a/CIME/hist_utils.py b/CIME/hist_utils.py index ab8b387e853..699c7aab969 100644 --- a/CIME/hist_utils.py +++ b/CIME/hist_utils.py @@ -2,23 +2,24 @@ Functions for actions pertaining to history files. """ +import filecmp import logging import os import re -import filecmp import shutil -from CIME.XML.standard_module_setup import * from CIME.config import Config +from CIME.core.exceptions import CIMEError from CIME.test_status import TEST_NO_BASELINES_COMMENT, TEST_STATUS_FILENAME from CIME.utils import ( + SharedArea, + expect, get_current_commit, get_timestamp, - safe_copy, - SharedArea, parse_test_name, + run_cmd, + safe_copy, ) -from CIME.core.exceptions import CIMEError logger = logging.getLogger(__name__) diff --git a/CIME/jenkins_generic_job.py b/CIME/jenkins_generic_job.py index e35a5d04e67..c140d08a9ea 100644 --- a/CIME/jenkins_generic_job.py +++ b/CIME/jenkins_generic_job.py @@ -1,8 +1,18 @@ +import glob +import logging +import os +import re +import shutil +import signal +import sys +import tarfile +import threading +import time + import CIME.wait_for_tests -from CIME.utils import expect, run_cmd_no_fail from CIME.case import Case +from CIME.utils import expect, run_cmd_no_fail -import os, shutil, glob, signal, logging, threading, sys, re, tarfile, time ############################################################################## def cleanup_queue(test_root, test_id): diff --git a/CIME/locked_files.py b/CIME/locked_files.py index 3be6f9c3573..91c2206543d 100644 --- a/CIME/locked_files.py +++ b/CIME/locked_files.py @@ -1,11 +1,12 @@ +import logging +import os from pathlib import Path -from CIME.utils import safe_copy -from CIME.XML.standard_module_setup import * +from CIME.utils import expect, safe_copy +from CIME.XML.env_batch import EnvBatch from CIME.XML.env_build import EnvBuild -from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.env_case import EnvCase -from CIME.XML.env_batch import EnvBatch +from CIME.XML.env_mach_pes import EnvMachPes from CIME.XML.generic_xml import GenericXML logger = logging.getLogger(__name__) diff --git a/CIME/namelist.py b/CIME/namelist.py index 486955bca69..e2a7cbdc93c 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -102,14 +102,15 @@ # (rather specific) pylint naming conventions. # pylint: disable=line-too-long,too-many-lines,invalid-name +import logging import re from contextlib import contextmanager +from CIME.utils import expect, string_in_list + # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, string_in_list logger = logging.getLogger(__name__) @@ -250,7 +251,7 @@ def get_fortran_variable_indices(varname, varlen=1, allow_any_len=False): (1, -1, 1) """ m = FORTRAN_NAME_REGEX.search(varname) - (minindex, maxindex, step) = (1, varlen, 1) + minindex, maxindex, step = (1, varlen, 1) if m.group(4) is not None: minindex = int(m.group(4)) @@ -893,7 +894,6 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): class Namelist(object): - """Class representing a Fortran namelist. Public methods: @@ -1184,7 +1184,6 @@ def get_group_variables(self, group_name): def write( self, out_file, groups=None, append=False, format_="nml", sorted_groups=True ): - """Write a the output data (normally fortran namelist) to the out_file As with `parse`, the `out_file` argument can be either a file name, or a @@ -1359,7 +1358,6 @@ def _write_nuopc(self, out_file, groups, sorted_groups): class _NamelistEOF(Exception): - """Exception thrown for an unexpected end-of-file in a namelist. This is an internal helper class, and should never be raised in a context @@ -1381,7 +1379,6 @@ def __str__(self): class _NamelistParseError(Exception): - """Exception thrown when namelist input has a syntax error. This is an internal helper class, and should never be raised in a context @@ -1403,7 +1400,6 @@ def __str__(self): class _NamelistParser(object): # pylint:disable=too-few-public-methods - """Class to validate and read from Fortran namelist input. This is intended to be an internal helper class and should not be used @@ -2222,7 +2218,7 @@ def _parse_name_and_values(self, allow_eof_end=False): break # and if it really is a literal, add it. values.append(literal) - (minindex, maxindex, step) = get_fortran_variable_indices( + minindex, maxindex, step = get_fortran_variable_indices( name, allow_any_len=True ) if (minindex > 1 or maxindex > minindex or step > 1) and maxindex > 0: diff --git a/CIME/nmlgen.py b/CIME/nmlgen.py index abfb5650c07..003ddc024e0 100644 --- a/CIME/nmlgen.py +++ b/CIME/nmlgen.py @@ -7,23 +7,24 @@ # pylint: disable=wildcard-import,unused-wildcard-import import datetime -import re import hashlib +import logging +import os +import re -from CIME.XML.standard_module_setup import * from CIME.namelist import ( Namelist, - parse, character_literal_to_string, - string_to_character_literal, - expand_literal_list, compress_literal_list, + expand_literal_list, merge_literal_lists, + parse, + string_to_character_literal, ) -from CIME.XML.namelist_definition import NamelistDefinition from CIME.utils import expect, safe_copy -from CIME.XML.stream import Stream from CIME.XML.grids import GRID_SEP +from CIME.XML.namelist_definition import NamelistDefinition +from CIME.XML.stream import Stream logger = logging.getLogger(__name__) @@ -66,7 +67,6 @@ class NamelistGenerator(object): - """Utility class for generating namelists for a given component.""" _streams_variables = [] diff --git a/CIME/provenance.py b/CIME/provenance.py index 849bb2757d1..23789033f0e 100644 --- a/CIME/provenance.py +++ b/CIME/provenance.py @@ -4,16 +4,18 @@ Library for saving build/run provenance. """ -from CIME.XML.standard_module_setup import * +import logging +import os +import sys + from CIME.utils import ( SharedArea, convert_to_babylonian_time, + expect, get_current_commit, run_cmd, ) -import sys - logger = logging.getLogger(__name__) diff --git a/CIME/scripts/configure b/CIME/scripts/configure index 73a326da2bf..5bb1d0685e4 100755 --- a/CIME/scripts/configure +++ b/CIME/scripts/configure @@ -24,9 +24,9 @@ real_file_dir = os.path.dirname(os.path.realpath(__file__)) cimeroot = os.path.abspath(os.path.join(real_file_dir, "..", "..")) sys.path.insert(0, cimeroot) +from CIME.BuildTools.configure import configure from CIME.Tools.standard_script_setup import * from CIME.utils import expect, get_model -from CIME.BuildTools.configure import configure from CIME.XML.machines import Machines logger = logging.getLogger(__name__) diff --git a/CIME/scripts/create_clone.py b/CIME/scripts/create_clone.py index 536ae73d232..23ba07b4402 100755 --- a/CIME/scripts/create_clone.py +++ b/CIME/scripts/create_clone.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 +import re +from argparse import RawTextHelpFormatter + +from CIME.case import Case from CIME.Tools.standard_script_setup import * from CIME.utils import expect -from CIME.case import Case -from argparse import RawTextHelpFormatter -import re logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args): ############################################################################### @@ -144,9 +146,11 @@ def _main_func(): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] nint = len(startval) diff --git a/CIME/scripts/create_newcase.py b/CIME/scripts/create_newcase.py index 879a5d167c8..1b219137da6 100755 --- a/CIME/scripts/create_newcase.py +++ b/CIME/scripts/create_newcase.py @@ -6,6 +6,10 @@ Script to create a new CIME Case Control System (CSS) experimental case. """ +from argparse import RawTextHelpFormatter + +from CIME.case import Case +from CIME.config import Config from CIME.Tools.standard_script_setup import * from CIME.utils import ( expect, @@ -13,12 +17,10 @@ get_cime_default_driver, get_src_root, ) -from CIME.config import Config -from CIME.case import Case -from argparse import RawTextHelpFormatter logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, cimeroot, description): ############################################################################### @@ -384,9 +386,11 @@ def _main_func(description=None): if user_mods_dirs is not None: user_mods_dirs = [ - os.path.abspath(one_user_mods_dir) - if os.path.isdir(one_user_mods_dir) - else one_user_mods_dir + ( + os.path.abspath(one_user_mods_dir) + if os.path.isdir(one_user_mods_dir) + else one_user_mods_dir + ) for one_user_mods_dir in user_mods_dirs ] diff --git a/CIME/scripts/create_test.py b/CIME/scripts/create_test.py index 051c7430592..f2abdf1b19f 100755 --- a/CIME/scripts/create_test.py +++ b/CIME/scripts/create_test.py @@ -26,28 +26,31 @@ If this tool is missing any feature that you need, please add an issue on https://github.com/ESMCI/cime """ + +import argparse +import glob +import math +from argparse import RawTextHelpFormatter + +from CIME import get_tests, utils +from CIME.case import Case +from CIME.config import Config +from CIME.test_scheduler import RUN_PHASE, TestScheduler +from CIME.test_utils import get_tests_from_xml from CIME.Tools.standard_script_setup import * -from CIME import get_tests -from CIME.test_scheduler import TestScheduler, RUN_PHASE -from CIME import utils from CIME.utils import ( - expect, - convert_to_seconds, compute_total_time, convert_to_babylonian_time, - run_cmd_no_fail, + convert_to_seconds, + expect, get_cime_config, + run_cmd_no_fail, ) -from CIME.config import Config from CIME.XML.machines import Machines -from CIME.case import Case -from CIME.test_utils import get_tests_from_xml -from argparse import RawTextHelpFormatter - -import argparse, math, glob logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/scripts/query_config.py b/CIME/scripts/query_config.py index 41b8d804578..79f201ef41d 100755 --- a/CIME/scripts/query_config.py +++ b/CIME/scripts/query_config.py @@ -6,19 +6,18 @@ information will be listed for each. """ +import argparse +import logging import os import sys -import logging -import argparse -from CIME.Tools.standard_script_setup import * from CIME import utils -from CIME.XML.files import Files +from CIME.config import Config from CIME.XML.component import Component from CIME.XML.compsets import Compsets +from CIME.XML.files import Files from CIME.XML.grids import Grids from CIME.XML.machines import Machines -from CIME.config import Config logger = logging.getLogger(__name__) diff --git a/CIME/scripts/query_testlists.py b/CIME/scripts/query_testlists.py index ad00733cd61..02e9386543a 100755 --- a/CIME/scripts/query_testlists.py +++ b/CIME/scripts/query_testlists.py @@ -14,13 +14,14 @@ All of the above support the various --xml-* arguments for subsetting which tests are included. """ -from CIME.Tools.standard_script_setup import * from CIME.test_utils import get_tests_from_xml, test_to_string -from CIME.XML.tests import Tests +from CIME.Tools.standard_script_setup import * from CIME.utils import expect +from CIME.XML.tests import Tests logger = logging.getLogger(__name__) + ############################################################################### def parse_command_line(args, description): ############################################################################### diff --git a/CIME/simple_compare.py b/CIME/simple_compare.py index f1292778157..f3c59cf6109 100644 --- a/CIME/simple_compare.py +++ b/CIME/simple_compare.py @@ -1,7 +1,9 @@ -import os, re +import os +import re from CIME.utils import expect + ############################################################################### def _normalize_string_value(value, case): ############################################################################### diff --git a/CIME/status.py b/CIME/status.py index 08618c7219d..e7093e4b2f7 100644 --- a/CIME/status.py +++ b/CIME/status.py @@ -1,6 +1,10 @@ # These routines were moved from utils.py to avoid circular dependancies -import time, os, sys, logging -from CIME.utils import Timeout, CASE_SUCCESS, CASE_FAILURE +import logging +import os +import sys +import time + +from CIME.utils import CASE_FAILURE, CASE_SUCCESS, Timeout logger = logging.getLogger(__name__) diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index d94a99e08fa..7b1152e1713 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -8,45 +8,67 @@ they can be run outside the context of TestScheduler. """ +import glob +import logging import os -import traceback, stat, threading, time, glob +import re +import stat +import sys +import threading +import time +import traceback from collections import OrderedDict -from CIME.XML.standard_module_setup import * -from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test +from CIME.build import post_build +from CIME.case import Case +from CIME.config import Config +from CIME.cs_status_creator import create_cs_status +from CIME.get_tests import get_build_groups, get_recommended_test_time, is_perf_test +from CIME.hist_utils import generate_teststatus +from CIME.locked_files import lock_file +from CIME.provenance import get_recommended_test_time_based_on_past from CIME.status import append_status, append_testlog +from CIME.SystemTests.test_mods import find_test_mods +from CIME.test_status import ( + CORE_PHASES, + CREATE_NEWCASE_PHASE, + MODEL_BUILD_PHASE, + RUN_PHASE, + SETUP_PHASE, + SHAREDLIB_BUILD_PHASE, + SUBMIT_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_RERUN_COMMENT, + TestStatus, + XML_PHASE, +) from CIME.utils import ( TESTS_FAILED_ERR_CODE, - parse_test_name, - get_full_test_name, - get_model, + CIMEError, convert_to_seconds, + expect, + get_cime_default_driver, get_cime_root, + get_full_test_name, + get_model, + get_project, get_src_root, - get_tools_path, get_template_path, - get_project, get_timestamp, - get_cime_default_driver, - CIMEError, + get_tools_path, + parse_test_name, + run_cmd, ) -from CIME.config import Config -from CIME.test_status import * -from CIME.XML.machines import Machines -from CIME.XML.generic_xml import GenericXML -from CIME.XML.env_test import EnvTest +from CIME.wait_for_tests import wait_for_tests +from CIME.XML.component import Component from CIME.XML.env_mach_pes import EnvMachPes +from CIME.XML.env_test import EnvTest from CIME.XML.files import Files -from CIME.XML.component import Component +from CIME.XML.generic_xml import GenericXML +from CIME.XML.machines import Machines from CIME.XML.tests import Tests -from CIME.case import Case -from CIME.wait_for_tests import wait_for_tests -from CIME.provenance import get_recommended_test_time_based_on_past -from CIME.locked_files import lock_file -from CIME.cs_status_creator import create_cs_status -from CIME.hist_utils import generate_teststatus -from CIME.build import post_build -from CIME.SystemTests.test_mods import find_test_mods logger = logging.getLogger(__name__) diff --git a/CIME/test_status.py b/CIME/test_status.py index 24ce5bc6e74..d3f4580403f 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -26,9 +26,12 @@ """ -from CIME.XML.standard_module_setup import * -import os, itertools +import itertools +import logging +import os + from CIME import expected_fails +from CIME.utils import expect TEST_STATUS_FILENAME = "TestStatus" diff --git a/CIME/test_utils.py b/CIME/test_utils.py index 6dea4e3bddc..c3a3c964da0 100644 --- a/CIME/test_utils.py +++ b/CIME/test_utils.py @@ -2,12 +2,16 @@ Utility functions used in test_scheduler.py, and by other utilities that need to get test lists. """ + import glob -from CIME.XML.standard_module_setup import * -from CIME.XML.testlist import Testlist -from CIME.XML.files import Files -from CIME.test_status import TEST_STATUS_FILENAME +import logging +import os + import CIME.utils +from CIME.test_status import TEST_STATUS_FILENAME +from CIME.utils import expect +from CIME.XML.files import Files +from CIME.XML.testlist import Testlist logger = logging.getLogger(__name__) diff --git a/CIME/tests/base.py b/CIME/tests/base.py index 8873924b60a..b2c4fea8bd9 100644 --- a/CIME/tests/base.py +++ b/CIME/tests/base.py @@ -2,17 +2,16 @@ import glob import os -import tempfile -import time -import signal import shutil +import signal import stat import sys +import tempfile +import time import unittest from CIME import utils from CIME.config import Config -from CIME.XML.machines import Machines def typed_os_environ(key, default_value, expected_type=None): diff --git a/CIME/tests/custom_assertions_test_status.py b/CIME/tests/custom_assertions_test_status.py index 16dc44e5cea..365b0f9d34b 100644 --- a/CIME/tests/custom_assertions_test_status.py +++ b/CIME/tests/custom_assertions_test_status.py @@ -3,11 +3,11 @@ can be used when testing TestStatus. """ -from CIME.XML.standard_module_setup import * - -import unittest import re +import unittest + from CIME import test_status +from CIME.utils import expect class CustomAssertionsTestStatus(unittest.TestCase): diff --git a/CIME/tests/scripts_regression_tests.py b/CIME/tests/scripts_regression_tests.py index 66f4c015298..7590e53e3cf 100755 --- a/CIME/tests/scripts_regression_tests.py +++ b/CIME/tests/scripts_regression_tests.py @@ -5,46 +5,31 @@ to confirm overall CIME correctness. """ -import glob, os, re, shutil, signal, sys, tempfile, threading, time, logging, unittest, getpass, filecmp, time, atexit, functools +import atexit +import functools +import logging +import os +import sys +import unittest CIMEROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) sys.path.insert(0, CIMEROOT) -from xml.etree.ElementTree import ParseError - -import subprocess, argparse +import argparse +import subprocess subprocess.call('/bin/rm -f $(find . -name "*.pyc")', shell=True, cwd=CIMEROOT) -import stat as osstat - -import collections +import CIME.test_scheduler +import CIME.wait_for_tests +from CIME import utils +from CIME.config import Config +from CIME.tests.base import BaseTestCase from CIME.utils import ( - run_cmd, - run_cmd_no_fail, - get_lids, get_current_commit, - safe_copy, - CIMEError, - get_cime_root, - get_src_root, - Timeout, - import_from_file, get_model, ) -import CIME.test_scheduler, CIME.wait_for_tests -from CIME import get_tests -from CIME.test_scheduler import TestScheduler -from CIME.XML.env_run import EnvRun from CIME.XML.machines import Machines -from CIME.XML.files import Files -from CIME.case import Case -from CIME.code_checker import check_code, get_all_checkable_files -from CIME.test_status import * -from CIME.provenance import get_test_success, save_test_success -from CIME import utils -from CIME.tests.base import BaseTestCase -from CIME.config import Config os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00" diff --git a/CIME/tests/test_sys_bless_tests_results.py b/CIME/tests/test_sys_bless_tests_results.py index 2f2beeb461d..9d8b061b466 100644 --- a/CIME/tests/test_sys_bless_tests_results.py +++ b/CIME/tests/test_sys_bless_tests_results.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import glob -import re import os +import re import stat from CIME import utils diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index c4095972d93..3b142118e53 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -8,10 +8,10 @@ import time from CIME import utils -from CIME.tests import base from CIME.case.case import Case -from CIME.XML.env_run import EnvRun +from CIME.tests import base from CIME.utils import find_system_test +from CIME.XML.env_run import EnvRun try: collectionsAbc = collections.abc diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index 1be636aff36..fce9743de09 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -7,9 +7,9 @@ import sys from CIME import utils -from CIME.tests import base -from CIME.case.case import Case from CIME.build import CmakeTmpBuildDir +from CIME.case.case import Case +from CIME.tests import base class TestCreateNewcase(base.BaseTestCase): diff --git a/CIME/tests/test_sys_full_system.py b/CIME/tests/test_sys_full_system.py index ad4485d2b81..d6e334839a7 100644 --- a/CIME/tests/test_sys_full_system.py +++ b/CIME/tests/test_sys_full_system.py @@ -2,10 +2,7 @@ import os -from CIME import get_tests -from CIME import test_status -from CIME import utils -from CIME import wait_for_tests +from CIME import get_tests, test_status, utils, wait_for_tests from CIME.tests import base diff --git a/CIME/tests/test_sys_jenkins_generic_job.py b/CIME/tests/test_sys_jenkins_generic_job.py index 30b31c5c8d6..19572717418 100644 --- a/CIME/tests/test_sys_jenkins_generic_job.py +++ b/CIME/tests/test_sys_jenkins_generic_job.py @@ -7,8 +7,7 @@ import threading import time -from CIME import get_tests -from CIME import utils +from CIME import get_tests, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_manage_and_query.py b/CIME/tests/test_sys_manage_and_query.py index 31ae3392bdb..f99b47aefb9 100644 --- a/CIME/tests/test_sys_manage_and_query.py +++ b/CIME/tests/test_sys_manage_and_query.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base from CIME.XML.files import Files diff --git a/CIME/tests/test_sys_save_timings.py b/CIME/tests/test_sys_save_timings.py index ee4a964c4b3..c5579489330 100644 --- a/CIME/tests/test_sys_save_timings.py +++ b/CIME/tests/test_sys_save_timings.py @@ -4,10 +4,9 @@ import glob import os -from CIME import provenance -from CIME import utils -from CIME.tests import base +from CIME import provenance, utils from CIME.case.case import Case +from CIME.tests import base class TestSaveTimings(base.BaseTestCase): diff --git a/CIME/tests/test_sys_single_submit.py b/CIME/tests/test_sys_single_submit.py index fed850263df..beeb80207b8 100644 --- a/CIME/tests/test_sys_single_submit.py +++ b/CIME/tests/test_sys_single_submit.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 1b72ed78d0b..df42ad14180 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -1,16 +1,13 @@ #!/usr/bin/env python3 -import re import glob import logging import os +import re import unittest from unittest import mock -from CIME import get_tests -from CIME import utils -from CIME import test_status -from CIME import test_scheduler +from CIME import get_tests, test_scheduler, test_status, utils from CIME.tests import base diff --git a/CIME/tests/test_sys_unittest.py b/CIME/tests/test_sys_unittest.py index 8bfd21eb16f..c22ff7d750c 100755 --- a/CIME/tests/test_sys_unittest.py +++ b/CIME/tests/test_sys_unittest.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os +import re import shutil import sys -import re from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_sys_wait_for_tests.py b/CIME/tests/test_sys_wait_for_tests.py index 3484c9af7a5..f0afd6d712f 100644 --- a/CIME/tests/test_sys_wait_for_tests.py +++ b/CIME/tests/test_sys_wait_for_tests.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 import os -import signal import shutil +import signal import sys -import time import threading +import time -from CIME import utils -from CIME import test_status -from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH +from CIME import test_status, utils from CIME.tests import base from CIME.tests import utils as test_utils +from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH class TestWaitForTests(base.BaseTestCase): @@ -338,9 +337,11 @@ def live_test_impl(self, testdir, expected_results, last_phase, last_status): ) as ts: ts.set_status( core_phase, - last_status - if core_phase == last_phase - else test_status.TEST_PASS_STATUS, + ( + last_status + if core_phase == last_phase + else test_status.TEST_PASS_STATUS + ), ) time.sleep(5) diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index 0ef5b387fa4..8f8e2e574d0 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -3,8 +3,8 @@ import gzip import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.baselines import performance from CIME.tests.test_unit_system_tests import CPLLOG @@ -300,7 +300,7 @@ def test_perf_compare_throughput_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -332,7 +332,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -364,7 +364,7 @@ def test_perf_compare_throughput_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -396,7 +396,7 @@ def test_perf_compare_throughput_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_throughput_baseline( + below_tolerance, comment = performance.perf_compare_throughput_baseline( case ) @@ -433,7 +433,7 @@ def test_perf_compare_memory_baseline_no_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -464,7 +464,7 @@ def test_perf_compare_memory_baseline_not_enough_samples( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance is None assert comment == "Found 2 memory usage samples, need atleast 4" @@ -524,7 +524,7 @@ def test_perf_compare_memory_baseline_no_tolerance( None, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( @@ -559,7 +559,7 @@ def test_perf_compare_memory_baseline_above_threshold( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert not below_tolerance assert ( @@ -594,7 +594,7 @@ def test_perf_compare_memory_baseline( 0.05, ) - (below_tolerance, comment) = performance.perf_compare_memory_baseline(case) + below_tolerance, comment = performance.perf_compare_memory_baseline(case) assert below_tolerance assert ( diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index 058c10caab8..63af7ecc140 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -1,21 +1,20 @@ import re -import unittest import tempfile +import unittest from contextlib import ExitStack -from unittest import mock from pathlib import Path +from unittest import mock from CIME.bless_test_results import ( - bless_test_results, - _bless_throughput, _bless_memory, + _bless_throughput, bless_history, bless_namelists, + bless_test_results, is_hist_bless_needed, ) from CIME.test_status import ALL_PHASES, GENERATE_PHASE from CIME.tests import utils as test_utils -from CIME.core.exceptions import CIMEError class TestUnitBlessTestResults(unittest.TestCase): diff --git a/CIME/tests/test_unit_build.py b/CIME/tests/test_unit_build.py index cb9ea6d3f81..0afce20bb0b 100644 --- a/CIME/tests/test_unit_build.py +++ b/CIME/tests/test_unit_build.py @@ -3,7 +3,6 @@ import os import unittest from unittest import mock -from pathlib import Path from CIME import build from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case.py b/CIME/tests/test_unit_case.py index 75fcbeed490..5cb73e198d4 100755 --- a/CIME/tests/test_unit_case.py +++ b/CIME/tests/test_unit_case.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 import os +import tempfile import unittest from unittest import mock -import tempfile -from CIME.case import case_submit -from CIME.case import Case from CIME import utils +from CIME.case import Case, case_submit from CIME.tests.utils import mock_case diff --git a/CIME/tests/test_unit_case_fake.py b/CIME/tests/test_unit_case_fake.py index 448931ecc7c..adc3252aa85 100755 --- a/CIME/tests/test_unit_case_fake.py +++ b/CIME/tests/test_unit_case_fake.py @@ -4,10 +4,10 @@ This module contains unit tests of CaseFake """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_case_run.py b/CIME/tests/test_unit_case_run.py index ce74f675366..3b869d6e49d 100644 --- a/CIME/tests/test_unit_case_run.py +++ b/CIME/tests/test_unit_case_run.py @@ -1,9 +1,8 @@ import unittest from unittest import mock +from CIME.case.case_run import TERMINATION_TEXT, _post_run_check from CIME.core.exceptions import CIMEError -from CIME.case.case_run import TERMINATION_TEXT -from CIME.case.case_run import _post_run_check def _case_post_run_check(): diff --git a/CIME/tests/test_unit_case_setup.py b/CIME/tests/test_unit_case_setup.py index a00dcf1b413..80ad93d8b46 100644 --- a/CIME/tests/test_unit_case_setup.py +++ b/CIME/tests/test_unit_case_setup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 +import contextlib import os -import unittest import tempfile -import contextlib +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/test_unit_case_st_archive.py b/CIME/tests/test_unit_case_st_archive.py index bf27ab1c201..5b72f8f8765 100644 --- a/CIME/tests/test_unit_case_st_archive.py +++ b/CIME/tests/test_unit_case_st_archive.py @@ -7,8 +7,8 @@ from CIME import date from CIME.case import case_st_archive -from CIME.XML import env_archive as _env_archive from CIME.tests import utils +from CIME.XML import env_archive as _env_archive UNSET = r""" diff --git a/CIME/tests/test_unit_compare_test_results.py b/CIME/tests/test_unit_compare_test_results.py index 4844a96c1a6..fe984390072 100755 --- a/CIME/tests/test_unit_compare_test_results.py +++ b/CIME/tests/test_unit_compare_test_results.py @@ -4,14 +4,18 @@ This module contains unit tests for compare_test_results """ -import unittest -import tempfile import os import shutil +import tempfile +import unittest -from CIME import utils -from CIME import compare_test_results -from CIME.test_status import * +from CIME import compare_test_results, utils +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + RUN_PHASE, + SETUP_PHASE, + TestStatus, +) from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_compare_two.py b/CIME/tests/test_unit_compare_two.py index e5f96001bc0..4cece00b660 100755 --- a/CIME/tests/test_unit_compare_two.py +++ b/CIME/tests/test_unit_compare_two.py @@ -9,16 +9,16 @@ # # pylint:disable=protected-access -import unittest -from collections import namedtuple import functools import os import shutil import tempfile +import unittest +from collections import namedtuple from unittest import mock -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo import CIME.test_status as test_status +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake # ======================================================================== diff --git a/CIME/tests/test_unit_config.py b/CIME/tests/test_unit_config.py index 93da359b046..f928b8a7479 100644 --- a/CIME/tests/test_unit_config.py +++ b/CIME/tests/test_unit_config.py @@ -1,8 +1,8 @@ import os -import unittest import tempfile -from unittest import mock +import unittest from pathlib import Path +from unittest import mock from CIME.config import Config diff --git a/CIME/tests/test_unit_configure.py b/CIME/tests/test_unit_configure.py index 693abccdfbd..7dc8971e2a3 100644 --- a/CIME/tests/test_unit_configure.py +++ b/CIME/tests/test_unit_configure.py @@ -1,11 +1,10 @@ import os -import unittest -import tempfile import shutil - -from CIME.utils import expect +import tempfile +import unittest from CIME.BuildTools.configure import generate_env_mach_specific +from CIME.utils import expect from CIME.XML.machines import Machines diff --git a/CIME/tests/test_unit_cs_status.py b/CIME/tests/test_unit_cs_status.py index 70efd47935f..1e44ce996e0 100755 --- a/CIME/tests/test_unit_cs_status.py +++ b/CIME/tests/test_unit_cs_status.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import io -import unittest -import shutil import os -import tempfile import re -from CIME.cs_status import cs_status +import shutil +import tempfile +import unittest + from CIME import test_status +from CIME.cs_status import cs_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_custom_assertions_test_status.py b/CIME/tests/test_unit_custom_assertions_test_status.py index 99e6cb05d04..4715377e5c8 100755 --- a/CIME/tests/test_unit_custom_assertions_test_status.py +++ b/CIME/tests/test_unit_custom_assertions_test_status.py @@ -5,6 +5,7 @@ """ import unittest + from CIME import test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_doctest.py b/CIME/tests/test_unit_doctest.py index 571b9af41db..1b972c0a6c9 100644 --- a/CIME/tests/test_unit_doctest.py +++ b/CIME/tests/test_unit_doctest.py @@ -1,17 +1,7 @@ #!/usr/bin/env python3 -import glob -import re import os -import stat -import doctest -import sys -import pkgutil -import unittest -import functools -import CIME -from CIME import utils from CIME.tests import base diff --git a/CIME/tests/test_unit_expected_fails_file.py b/CIME/tests/test_unit_expected_fails_file.py index 30c9dd3a6e8..d2898a5022e 100755 --- a/CIME/tests/test_unit_expected_fails_file.py +++ b/CIME/tests/test_unit_expected_fails_file.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile -from CIME.XML.expected_fails_file import ExpectedFailsFile +import unittest + from CIME.core.exceptions import CIMEError from CIME.expected_fails import ExpectedFails +from CIME.XML.expected_fails_file import ExpectedFailsFile class TestExpectedFailsFile(unittest.TestCase): diff --git a/CIME/tests/test_unit_fake_case.py b/CIME/tests/test_unit_fake_case.py index b7b20ac6abc..47e6fe88c5f 100755 --- a/CIME/tests/test_unit_fake_case.py +++ b/CIME/tests/test_unit_fake_case.py @@ -5,12 +5,12 @@ This is seperate from FakeCase that's under CIME/tests """ -import unittest import os -from CIME.core.exceptions import CIMEError -from CIME.utils import get_model, GLOBAL, get_src_root +import unittest from CIME.BuildTools.configure import FakeCase +from CIME.core.exceptions import CIMEError +from CIME.utils import get_model, get_src_root class TestFakeCase(unittest.TestCase): diff --git a/CIME/tests/test_unit_grids.py b/CIME/tests/test_unit_grids.py index e33349d52b5..2e2da3878b5 100755 --- a/CIME/tests/test_unit_grids.py +++ b/CIME/tests/test_unit_grids.py @@ -13,13 +13,14 @@ # # pylint:disable=line-too-long -import unittest import os import shutil import string import tempfile -from CIME.XML.grids import Grids, _ComponentGrids, _add_grid_info, _strip_grid_from_name +import unittest + from CIME.core.exceptions import CIMEError +from CIME.XML.grids import Grids, _add_grid_info, _ComponentGrids, _strip_grid_from_name class TestGrids(unittest.TestCase): diff --git a/CIME/tests/test_unit_hist_utils.py b/CIME/tests/test_unit_hist_utils.py index fe6d4866c34..86fcc5a9c95 100644 --- a/CIME/tests/test_unit_hist_utils.py +++ b/CIME/tests/test_unit_hist_utils.py @@ -1,9 +1,7 @@ -import io import unittest from unittest import mock from CIME.hist_utils import copy_histfiles -from CIME.XML.archive import Archive class TestHistUtils(unittest.TestCase): diff --git a/CIME/tests/test_unit_locked_files.py b/CIME/tests/test_unit_locked_files.py index 2732f16f3f0..abd858433b2 100644 --- a/CIME/tests/test_unit_locked_files.py +++ b/CIME/tests/test_unit_locked_files.py @@ -1,13 +1,11 @@ import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME import locked_files from CIME.core.exceptions import CIMEError -from CIME.XML.entry_id import EntryID from CIME.XML.env_batch import EnvBatch -from CIME.XML.files import Files def create_batch_system(env_batch, batch_submit_value=None): diff --git a/CIME/tests/test_unit_nmlgen.py b/CIME/tests/test_unit_nmlgen.py index 52e04d28856..76c274871be 100644 --- a/CIME/tests/test_unit_nmlgen.py +++ b/CIME/tests/test_unit_nmlgen.py @@ -1,10 +1,11 @@ -from collections import OrderedDict import tempfile import unittest +from collections import OrderedDict from unittest import mock from CIME.nmlgen import NamelistGenerator + # pylint: disable=protected-access class TestNamelistGenerator(unittest.TestCase): def test_init_defaults(self): diff --git a/CIME/tests/test_unit_paramgen.py b/CIME/tests/test_unit_paramgen.py index 26b2be4ac6e..698a77093a1 100755 --- a/CIME/tests/test_unit_paramgen.py +++ b/CIME/tests/test_unit_paramgen.py @@ -13,8 +13,9 @@ # # pylint:disable=line-too-long -import unittest import tempfile +import unittest + from CIME.ParamGen.paramgen import ParamGen ############### diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index c95c7fd2d9a..f2bbff1924f 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -1,18 +1,17 @@ #!/usr/bin/env python3 -import os -import tempfile import gzip +import os import re -from re import A +import tempfile import unittest -from unittest import mock from pathlib import Path +from unittest import mock from CIME.config import Config from CIME.SystemTests.system_tests_common import SystemTestsCommon -from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo CPLLOG = """ tStamp_write: model date = 00010102 0 wall clock = 2023-09-19 19:39:42 avg dt = 0.33 dt = 0.33 diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 0bb33f8d605..3bcf4a1713d 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -import os -import json -import unittest -import tempfile import contextlib +import json +import os import sysconfig +import tempfile +import unittest from pathlib import Path from unittest import mock + from CIME.tests.utils import chdir evv4esm = False diff --git a/CIME/tests/test_unit_test_status.py b/CIME/tests/test_unit_test_status.py index 9b3036801fc..061e2cafe40 100755 --- a/CIME/tests/test_unit_test_status.py +++ b/CIME/tests/test_unit_test_status.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import unittest import os -from CIME import test_status -from CIME import expected_fails +import unittest + +from CIME import expected_fails, test_status from CIME.tests.custom_assertions_test_status import CustomAssertionsTestStatus diff --git a/CIME/tests/test_unit_two_link_to_case2_output.py b/CIME/tests/test_unit_two_link_to_case2_output.py index 2860db08ac1..5d59a6ca298 100755 --- a/CIME/tests/test_unit_two_link_to_case2_output.py +++ b/CIME/tests/test_unit_two_link_to_case2_output.py @@ -10,10 +10,11 @@ # # pylint:disable=protected-access -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo from CIME.tests.case_fake import CaseFake diff --git a/CIME/tests/test_unit_user_mod_support.py b/CIME/tests/test_unit_user_mod_support.py index c2d0fc37e13..ca7289f1596 100755 --- a/CIME/tests/test_unit_user_mod_support.py +++ b/CIME/tests/test_unit_user_mod_support.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -import unittest +import os import shutil import tempfile -import os -from CIME.user_mod_support import apply_user_mods +import unittest + from CIME.core.exceptions import CIMEError +from CIME.user_mod_support import apply_user_mods # ======================================================================== # Define some parameters diff --git a/CIME/tests/test_unit_user_nl_utils.py b/CIME/tests/test_unit_user_nl_utils.py index 9220182eeeb..2ec199874b8 100755 --- a/CIME/tests/test_unit_user_nl_utils.py +++ b/CIME/tests/test_unit_user_nl_utils.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import unittest import os import shutil import tempfile +import unittest + from CIME.SystemTests.test_utils import user_nl_utils diff --git a/CIME/tests/test_unit_utils.py b/CIME/tests/test_unit_utils.py index 8abde4d8fef..00f8fd1a047 100755 --- a/CIME/tests/test_unit_utils.py +++ b/CIME/tests/test_unit_utils.py @@ -1,21 +1,20 @@ #!/usr/bin/env python3 import os -import stat import shutil import sys import tempfile - import unittest from unittest import mock + from CIME.status import run_and_log_case_status from CIME.utils import ( - indent_string, - import_from_file, _line_defines_python_function, - file_contains_python_function, copy_globs, + file_contains_python_function, import_and_run_sub_or_cmd, + import_from_file, + indent_string, ) diff --git a/CIME/tests/test_unit_xml_archive_base.py b/CIME/tests/test_unit_xml_archive_base.py index 98f58055c9c..42789333dc0 100644 --- a/CIME/tests/test_unit_xml_archive_base.py +++ b/CIME/tests/test_unit_xml_archive_base.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -import os import io -import unittest import tempfile +import unittest from contextlib import contextmanager from pathlib import Path -from unittest import mock from CIME.XML.archive_base import ArchiveBase diff --git a/CIME/tests/test_unit_xml_env_batch.py b/CIME/tests/test_unit_xml_env_batch.py index 5b1295c4a68..b3777031d27 100755 --- a/CIME/tests/test_unit_xml_env_batch.py +++ b/CIME/tests/test_unit_xml_env_batch.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import os -import unittest import tempfile +import unittest from contextlib import ExitStack from unittest import mock +from CIME.BuildTools.configure import FakeCase from CIME.core.exceptions import CIMEError from CIME.utils import expect from CIME.XML.env_batch import EnvBatch, get_job_deps from CIME.XML.env_workflow import EnvWorkflow -from CIME.BuildTools.configure import FakeCase # pylint: disable=unused-argument diff --git a/CIME/tests/test_unit_xml_env_mach_specific.py b/CIME/tests/test_unit_xml_env_mach_specific.py index a900034fad8..ef72a061ac5 100644 --- a/CIME/tests/test_unit_xml_env_mach_specific.py +++ b/CIME/tests/test_unit_xml_env_mach_specific.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import unittest import tempfile +import unittest from unittest import mock from CIME import utils diff --git a/CIME/tests/test_unit_xml_grids.py b/CIME/tests/test_unit_xml_grids.py index cceae5b419d..292012033bc 100644 --- a/CIME/tests/test_unit_xml_grids.py +++ b/CIME/tests/test_unit_xml_grids.py @@ -1,10 +1,6 @@ import os -import io -import unittest import tempfile -from contextlib import contextmanager -from pathlib import Path -from unittest import mock +import unittest from CIME.core.exceptions import CIMEError from CIME.XML.grids import Grids diff --git a/CIME/tests/test_unit_xml_tests.py b/CIME/tests/test_unit_xml_tests.py index 898e00bb0b8..23d5d8fac4c 100644 --- a/CIME/tests/test_unit_xml_tests.py +++ b/CIME/tests/test_unit_xml_tests.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import re -import unittest import tempfile +import unittest from pathlib import Path from unittest import mock diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 5987f0fd18e..d74d0798f44 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -1,19 +1,16 @@ +import contextlib import io import os -import tempfile -import signal import shutil import sys -import time -import contextlib +import tempfile +import xml.etree.ElementTree as ET from collections.abc import Iterable from unittest import mock -import xml.etree.ElementTree as ET -from CIME import utils from CIME import test_status -from CIME.utils import expect from CIME.case import Case +from CIME.utils import expect from CIME.XML.entry_id import EntryID MACRO_PRESERVE_ENV = [ @@ -321,7 +318,6 @@ def get_default_compiler(self): class MakefileTester(object): - """Helper class for checking Makefile output. Public methods: @@ -421,7 +417,6 @@ def assert_variable_matches(self, var_name, regex, env=None, var=None): class CMakeTester(object): - """Helper class for checking CMake output. Public methods: diff --git a/CIME/user_mod_support.py b/CIME/user_mod_support.py index 2c182a19dbd..b4eeec21c1a 100644 --- a/CIME/user_mod_support.py +++ b/CIME/user_mod_support.py @@ -2,9 +2,11 @@ user_mod_support.py """ -from CIME.XML.standard_module_setup import * -from CIME.utils import expect, run_cmd_no_fail, safe_copy import glob +import logging +import os + +from CIME.utils import expect, run_cmd_no_fail, safe_copy logger = logging.getLogger(__name__) diff --git a/CIME/utils.py b/CIME/utils.py index 948632b4068..52d38c76ed8 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -3,12 +3,27 @@ Warning: you cannot use CIME Classes in this module as it causes circular dependencies """ -import shlex import configparser -import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, importlib, fnmatch +import errno +import filecmp +import fnmatch +import glob +import gzip +import importlib import importlib.util -import errno, signal, warnings, filecmp +import io +import logging +import os +import random +import re +import shlex +import shutil +import signal import stat as statlib +import string +import sys +import time +import warnings from argparse import Action from contextlib import contextmanager diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index a32278ed052..51b53ce7f63 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -1,18 +1,36 @@ # pylint: disable=import-error -import queue, os, time, threading, socket, signal, shutil, glob, tempfile -from pathlib import Path +import glob # pylint: disable=import-error import logging +import os +import queue +import shutil +import signal +import socket +import tempfile +import threading +import time import xml.etree.ElementTree as xmlet +from pathlib import Path import CIME.utils +from CIME.case.case import Case from CIME.core.exceptions import CIMEError -from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy -from CIME.XML.machines import Machines -from CIME.test_status import * from CIME.provenance import save_test_success -from CIME.case.case import Case +from CIME.test_status import ( + CREATE_NEWCASE_PHASE, + NAMELIST_FAIL_STATUS, + NAMELIST_PHASE, + RUN_PHASE, + TEST_FAIL_STATUS, + TEST_PASS_STATUS, + TEST_PEND_STATUS, + TEST_STATUS_FILENAME, + TestStatus, +) +from CIME.utils import Timeout, expect, run_cmd, run_cmd_no_fail, safe_copy +from CIME.XML.machines import Machines SIGNAL_RECEIVED = False E3SM_MAIN_CDASH = "E3SM"