From 06e2b53665b5d6accd968fea6fb848c8265631db Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 14 Jul 2025 16:29:16 +0200 Subject: [PATCH 1/8] Concentrations are defined in a default area, then removed for each river --- BC/EFAS/rivers.xml | 674 ++------------------------------------------- 1 file changed, 19 insertions(+), 655 deletions(-) diff --git a/BC/EFAS/rivers.xml b/BC/EFAS/rivers.xml index e29c13b..0cb47f2 100644 --- a/BC/EFAS/rivers.xml +++ b/BC/EFAS/rivers.xml @@ -128,15 +128,13 @@ - - - - - - - - + + + + + + @@ -157,6 +155,19 @@ + + + + + + + + + + + + + @@ -173,23 +184,6 @@ - - - - - - - - - - - - - - - - - @@ -206,23 +200,6 @@ - - - - - - - - - - - - - - - - - @@ -248,23 +225,6 @@ - - - - - - - - - - - - - - - - - @@ -282,23 +242,6 @@ - - - - - - - - - - - - - - - - - @@ -315,23 +258,6 @@ - - - - - - - - - - - - - - - - - @@ -346,23 +272,6 @@ - - - - - - - - - - - - - - - - - @@ -380,23 +289,6 @@ - - - - - - - - - - - - - - - - - @@ -411,23 +303,6 @@ - - - - - - - - - - - - - - - - - @@ -442,23 +317,6 @@ - - - - - - - - - - - - - - - - - @@ -473,23 +331,6 @@ - - - - - - - - - - - - - - - - - @@ -504,23 +345,6 @@ - - - - - - - - - - - - - - - - - @@ -535,23 +359,6 @@ - - - - - - - - - - - - - - - - - @@ -566,23 +373,6 @@ - - - - - - - - - - - - - - - - - @@ -597,23 +387,6 @@ - - - - - - - - - - - - - - - - - @@ -628,23 +401,6 @@ - - - - - - - - - - - - - - - - - @@ -659,23 +415,6 @@ - - - - - - - - - - - - - - - - - @@ -690,23 +429,6 @@ - - - - - - - - - - - - - - - - - @@ -721,23 +443,6 @@ - - - - - - - - - - - - - - - - - @@ -752,23 +457,6 @@ - - - - - - - - - - - - - - - - - @@ -783,23 +471,6 @@ - - - - - - - - - - - - - - - - - @@ -817,23 +488,6 @@ - - - - - - - - - - - - - - - - - @@ -848,23 +502,6 @@ - - - - - - - - - - - - - - - - - @@ -879,23 +516,6 @@ - - - - - - - - - - - - - - - - - @@ -909,24 +529,6 @@ - - - - - - - - - - - - - - - - - - @@ -941,23 +543,6 @@ - - - - - - - - - - - - - - - - - @@ -972,23 +557,6 @@ - - - - - - - - - - - - - - - - - @@ -1003,23 +571,6 @@ - - - - - - - - - - - - - - - - - @@ -1034,23 +585,6 @@ - - - - - - - - - - - - - - - - - @@ -1065,23 +599,6 @@ - - - - - - - - - - - - - - - - - @@ -1096,23 +613,6 @@ - - - - - - - - - - - - - - - - - @@ -1127,23 +627,6 @@ - - - - - - - - - - - - - - - - - @@ -1158,23 +641,6 @@ - - - - - - - - - - - - - - - - - @@ -1189,23 +655,6 @@ - - - - - - - - - - - - - - - - - @@ -1220,23 +669,6 @@ - - - - - - - - - - - - - - - - - @@ -1251,23 +683,6 @@ - - - - - - - - - - - - - - - - - @@ -1282,23 +697,6 @@ - - - - - - - - - - - - - - - - - @@ -1313,23 +711,6 @@ - - - - - - - - - - - - - - - - - @@ -1344,23 +725,6 @@ - - - - - - - - - - - - - - - - - From ab26bc0a49ebe4086d320f0a24082bff3522619d Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 14 Jul 2025 17:40:13 +0200 Subject: [PATCH 2/8] river_reader.py reads and applies commons_concentrations from xml file --- BC/EFAS/river_reader.py | 18 ++++++++++++++++++ BC/EFAS/rivers.xml | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/BC/EFAS/river_reader.py b/BC/EFAS/river_reader.py index c810648..e6ea75b 100644 --- a/BC/EFAS/river_reader.py +++ b/BC/EFAS/river_reader.py @@ -59,8 +59,21 @@ def read_xml_vars(xml_dataset): return tuple(bgc_vars) +def read_commons(xml_dataset): + """ + Read the field "common_concentrations" of the rivers.xml file + """ + commons_node=xmldoc.getElementsByTagName("common_concentrations") + commons={} + for node in commons_node[0].getElementsByTagName("var"): + var_name = node.getAttribute("name") + value = float(node.getAttribute("value")) + commons[var_name]=value + return commons + xmldoc = minidom.parse("rivers.xml") BGC_VARS = read_xml_vars(xmldoc) +COMMONS = read_commons(xmldoc) class Rivers: @@ -134,6 +147,11 @@ def __init__(self): rivers[ind]['SAL'] = salinity rivers[ind]['name'] = name rivers[ind]['mouth'] = mn.getAttribute('name') + for varname in COMMONS: + if varname not in self.__bgc_var_dict: + raise ValueError('Unknown variable: {}'.format(varname)) + rivers[ind][varname] = COMMONS[varname] + for vn in concentrations_node.getElementsByTagName('var'): varname = vn.getAttribute('name') if varname not in self.__bgc_var_dict: diff --git a/BC/EFAS/rivers.xml b/BC/EFAS/rivers.xml index 0cb47f2..43cd947 100644 --- a/BC/EFAS/rivers.xml +++ b/BC/EFAS/rivers.xml @@ -128,7 +128,7 @@ - + @@ -154,7 +154,7 @@ - + From ee1a7d3ccc158b020bbd0dec35f0dbcd0136f1d1 Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 14 Jul 2025 17:46:15 +0200 Subject: [PATCH 3/8] Taking O2o and R3l from commons section of rivers.xml --- BC/EFAS/rivers.xml | 79 +--------------------------------------------- 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/BC/EFAS/rivers.xml b/BC/EFAS/rivers.xml index 43cd947..39adb03 100644 --- a/BC/EFAS/rivers.xml +++ b/BC/EFAS/rivers.xml @@ -165,8 +165,6 @@ - - @@ -182,8 +180,6 @@ - - @@ -198,8 +194,6 @@ - - @@ -223,8 +217,6 @@ - - @@ -240,8 +232,6 @@ - - @@ -256,8 +246,6 @@ - - @@ -270,8 +258,6 @@ - - @@ -287,8 +273,6 @@ - - @@ -301,8 +285,6 @@ - - @@ -315,8 +297,6 @@ - - @@ -329,8 +309,6 @@ - - @@ -342,9 +320,7 @@ - - - +> @@ -357,8 +333,6 @@ - - @@ -371,8 +345,6 @@ - - @@ -385,8 +357,6 @@ - - @@ -399,8 +369,6 @@ - - @@ -413,8 +381,6 @@ - - @@ -427,8 +393,6 @@ - - @@ -441,8 +405,6 @@ - - @@ -455,8 +417,6 @@ - - @@ -469,8 +429,6 @@ - - @@ -486,8 +444,6 @@ - - @@ -500,8 +456,6 @@ - - @@ -514,8 +468,6 @@ - - @@ -528,7 +480,6 @@ - @@ -541,8 +492,6 @@ - - @@ -555,8 +504,6 @@ - - @@ -569,8 +516,6 @@ - - @@ -583,8 +528,6 @@ - - @@ -597,8 +540,6 @@ - - @@ -611,8 +552,6 @@ - - @@ -625,8 +564,6 @@ - - @@ -639,8 +576,6 @@ - - @@ -653,8 +588,6 @@ - - @@ -667,8 +600,6 @@ - - @@ -681,8 +612,6 @@ - - @@ -695,8 +624,6 @@ - - @@ -709,8 +636,6 @@ - - @@ -723,8 +648,6 @@ - - From c8b09521b6f141758af776013d34716ca41e1e91 Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 14 Jul 2025 17:51:44 +0200 Subject: [PATCH 4/8] Appling factor 2.0 to all P?? variables --- BC/EFAS/rivers.xml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/BC/EFAS/rivers.xml b/BC/EFAS/rivers.xml index 39adb03..142b132 100644 --- a/BC/EFAS/rivers.xml +++ b/BC/EFAS/rivers.xml @@ -137,23 +137,23 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + From a608301330fc9ac11159b73c215123832389de52 Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Tue, 15 Jul 2025 14:27:12 +0200 Subject: [PATCH 5/8] Renaming commons_concentrations --> default_concentrations and improving robustness of xml reading --- BC/EFAS/river_reader.py | 29 ++++++++++++++++++----------- BC/EFAS/rivers.xml | 4 ++-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/BC/EFAS/river_reader.py b/BC/EFAS/river_reader.py index e6ea75b..34fe7c9 100644 --- a/BC/EFAS/river_reader.py +++ b/BC/EFAS/river_reader.py @@ -59,21 +59,28 @@ def read_xml_vars(xml_dataset): return tuple(bgc_vars) -def read_commons(xml_dataset): +def read_defaults(xml_dataset): """ Read the field "common_concentrations" of the rivers.xml file """ - commons_node=xmldoc.getElementsByTagName("common_concentrations") - commons={} - for node in commons_node[0].getElementsByTagName("var"): - var_name = node.getAttribute("name") - value = float(node.getAttribute("value")) - commons[var_name]=value - return commons + default_node=xml_dataset.getElementsByTagName("default_concentrations") + defaults={} + for node in default_node[0].getElementsByTagName("var"): + try: + var_name = node.getAttribute("name") + value = float(node.getAttribute("value")) + defaults[var_name]=value + except Exception: + raise InvalidRiverXMLFile( + 'Error while reading the following var node:\n{}'.format( + node.toxml() + ) + ) + return defaults xmldoc = minidom.parse("rivers.xml") BGC_VARS = read_xml_vars(xmldoc) -COMMONS = read_commons(xmldoc) +DEFAULTS = read_defaults(xmldoc) class Rivers: @@ -147,10 +154,10 @@ def __init__(self): rivers[ind]['SAL'] = salinity rivers[ind]['name'] = name rivers[ind]['mouth'] = mn.getAttribute('name') - for varname in COMMONS: + for varname in DEFAULTS: if varname not in self.__bgc_var_dict: raise ValueError('Unknown variable: {}'.format(varname)) - rivers[ind][varname] = COMMONS[varname] + rivers[ind][varname] = DEFAULTS[varname] for vn in concentrations_node.getElementsByTagName('var'): varname = vn.getAttribute('name') diff --git a/BC/EFAS/rivers.xml b/BC/EFAS/rivers.xml index 142b132..e97fda8 100644 --- a/BC/EFAS/rivers.xml +++ b/BC/EFAS/rivers.xml @@ -128,7 +128,7 @@ - + @@ -154,7 +154,7 @@ - + From e876e4aae7d9d4c43f3f344e25360bc2f66727ca Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 18 Aug 2025 14:35:28 +0200 Subject: [PATCH 6/8] Bug fix, Rivers.__var_defined field must be set to True also for defaults. If not, defaults will be not dumped in NetCDF output files. The check on xml is now done with an assertion + an if, it can be reviewed. --- BC/EFAS/river_reader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/BC/EFAS/river_reader.py b/BC/EFAS/river_reader.py index 34fe7c9..688c75d 100644 --- a/BC/EFAS/river_reader.py +++ b/BC/EFAS/river_reader.py @@ -159,6 +159,10 @@ def __init__(self): raise ValueError('Unknown variable: {}'.format(varname)) rivers[ind][varname] = DEFAULTS[varname] + bgc_var = self.__bgc_var_dict[varname] + assert not self.__var_defined[bgc_var][ind] + self.__var_defined[bgc_var][ind] = True + for vn in concentrations_node.getElementsByTagName('var'): varname = vn.getAttribute('name') if varname not in self.__bgc_var_dict: @@ -167,8 +171,8 @@ def __init__(self): rivers[ind][varname] = vn.getAttribute('value') - assert not self.__var_defined[bgc_var][ind] - self.__var_defined[bgc_var][ind] = True + if not self.__var_defined[bgc_var][ind]: + self.__var_defined[bgc_var][ind] = True visited_ids[ind] = True From 4ecc69862397c25dc6e2ef427188a4538423def2 Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Mon, 18 Aug 2025 15:02:35 +0200 Subject: [PATCH 7/8] Adding a check on var_mask, to avoid empty arrays --- BC/EFAS/river_generation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/BC/EFAS/river_generation.py b/BC/EFAS/river_generation.py index e02fddf..fdd7c11 100644 --- a/BC/EFAS/river_generation.py +++ b/BC/EFAS/river_generation.py @@ -146,6 +146,11 @@ def main(): ) var_mask = RIVERS.get_variable_mask(variable) + if not np.any(var_mask): + # If var_mask is False for any river, print a message + print(f"Warning: {variable.name} empty, it is not defined for any river") + + raw_var_concentration = RIVERS[variable.name][var_mask] # Keep only the values related to the current variable From 47a7cdb19d42b5c7e8214c5baef767cafb19c163 Mon Sep 17 00:00:00 2001 From: Giorgio Bolzon Date: Wed, 15 Oct 2025 15:10:41 +0200 Subject: [PATCH 8/8] ATMOSPHERE/online/ python scripts take now fv12 -> fv19 versions of ECMWF --- ATMOSPHERE/online/analysis_input_gen.py | 2 +- ATMOSPHERE/online/analysis_input_gen_singleframe.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ATMOSPHERE/online/analysis_input_gen.py b/ATMOSPHERE/online/analysis_input_gen.py index 7ad7100..3bb7522 100644 --- a/ATMOSPHERE/online/analysis_input_gen.py +++ b/ATMOSPHERE/online/analysis_input_gen.py @@ -53,7 +53,7 @@ def argument(): INPUTDIR=addsep(args.inputdir) OUTDIR = addsep(args.outdir) -TL = TimeList.fromfilenames(None, INPUTDIR, "*_an-fv12.00.nc", prefix="", dateformat="%Y%m%d") +TL = TimeList.fromfilenames(None, INPUTDIR, "*_an-fv1[2-9].00.nc", prefix="", dateformat="%Y%m%d") filename=TL.filelist[0] ncIN=netCDF4.Dataset(filename) diff --git a/ATMOSPHERE/online/analysis_input_gen_singleframe.py b/ATMOSPHERE/online/analysis_input_gen_singleframe.py index 0bb36fd..e63c534 100644 --- a/ATMOSPHERE/online/analysis_input_gen_singleframe.py +++ b/ATMOSPHERE/online/analysis_input_gen_singleframe.py @@ -4,7 +4,7 @@ def argument(): parser = argparse.ArgumentParser(description = ''' Generates daily files for OASIM by reading ECMWF file provided by CMCC Valid for analysis file having a single frame - YYYYMMDD-ECMWF---AM0100-MEDATL-bYYYYMMDD_an00-fv12.00.nc analisi at 00 of the same day YYYYMMDD + YYYYMMDD-ECMWF---AM0100-MEDATL-bYYYYMMDD_an00-fv1[2-9].00.nc analisi at 00 of the same day YYYYMMDD ''', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) @@ -48,7 +48,7 @@ def argument(): INPUTDIR=addsep(args.inputdir) OUTDIR = addsep(args.outdir) -TL = TimeList.fromfilenames(None, INPUTDIR, "*_an00-fv12.00.nc", prefix="", dateformat="%Y%m%d") +TL = TimeList.fromfilenames(None, INPUTDIR, "*_an00-fv1[2-9].00.nc", prefix="", dateformat="%Y%m%d") nframes_in_day = 1 deltaH = 6