remove any occurrence of sys.exit(); set to custom exception; removed redundant calculations for masking in intensification and expansion methods

Author: crvernon

demeter/change/expansion.py

@@ -109,13 +109,13 @@ def _expansion(diagnostic, diag_file, spat_ludataharm_sub, kernel_vector_sub, co
     trans_mat = np.zeros((l_shs, l_ord, l_ord))
 
     # process PFTs in order
-    for pft_ord in sorted(order_rules):
+    for pft_ord in np.unique(order_rules):
 
         # create a copy of the cons_data_sub array
         cons_data_sub = cons_data_sub_o.copy()
 
-        # lookup PFT and land class name
-        pft = order_rules.index(pft_ord)
+        # lookup PFT and final land class
+        pft = np.where(order_rules == pft_ord)[0][0]
         fcs = final_landclasses[pft]
 
         # define intensification targets
@@ -237,22 +237,23 @@ def apply_expansion(log, c, allregnumber, allregmet, spat_ludataharm, spat_regio
 
         # get region and metric from index
         regnumber = allregnumber[reg_idx]
-        metnumber = np.unique(spat_met)[met_idx]
-        met_ref = met_idx
+        metnumber = allregmet[reg_idx][met_idx]
+        metnum_idx = metnumber - 1
 
         # create data subset
-        spat_ludataharm_sub = spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)]
-        kernel_vector_sub = kernel_vector[(spat_region == regnumber) & (spat_met == metnumber)]
-        cons_data_sub = cons_data[(spat_region == regnumber) & (spat_met == metnumber)]
+        reg_met_mask = (spat_region == regnumber) & (spat_met == metnumber)
+        spat_ludataharm_sub = spat_ludataharm[reg_met_mask]
+        kernel_vector_sub = kernel_vector[reg_met_mask]
+        cons_data_sub = cons_data[reg_met_mask]
 
         # calculate expansion for each PFT
         exp = _expansion(c.diagnostic, diag_file, spat_ludataharm_sub, kernel_vector_sub, cons_data_sub, reg_idx,
-                         met_ref, order_rules, final_landclasses, c.errortol, constrain_rules, transition_rules,
+                         metnum_idx, order_rules, final_landclasses, c.errortol, constrain_rules, transition_rules,
                          c.stochastic_expansion, c.selection_threshold, land_mismatch, target_change)
 
         # apply expansion and update transitions
-        spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)], target_change, trans_mat = exp
-        transitions[(spat_region == regnumber) & (spat_met == metnumber), :, :] += trans_mat
+        spat_ludataharm[reg_met_mask], target_change, trans_mat = exp
+        transitions[reg_met_mask, :, :] += trans_mat
 
     # calculate non-achieved change
     non_chg = np.sum(abs(target_change[:, :, :])) / 2.

demeter/change/intensification.py

@@ -92,11 +92,12 @@ def _convert_pft(notdone, int_target, metnumber, pft_toconv, spat_ludataharm_sub
         spat_ludataharm_sub[exist_cells, pft_toconv] -= actexpansion
 
         # update the target change values
-        target_change[reg, metnumber - 1, pft] -= np.sum(actexpansion)
-        int_target -= np.sum(actexpansion)
-        target_intensification[metnumber - 1, pft] -= np.sum(actexpansion)
-        target_change[reg, metnumber - 1, pft_toconv] += np.sum(actexpansion)
-        target_intensification[metnumber - 1, pft_toconv] += np.sum(actexpansion)
+        actual_expansion_sum = np.sum(actexpansion)
+        target_change[reg, metnumber - 1, pft] -= actual_expansion_sum
+        int_target -= actual_expansion_sum
+        target_intensification[metnumber - 1, pft] -= actual_expansion_sum
+        target_change[reg, metnumber - 1, pft_toconv] += actual_expansion_sum
+        target_intensification[metnumber - 1, pft_toconv] += actual_expansion_sum
         trans_mat[exist_cells, pft, pft_toconv] += actexpansion
 
         # account for target change minuscule values when evaluating notdone
@@ -312,20 +313,21 @@ def apply_intensification(log, pass_number, c, spat_region, order_rules, allregn
         metnumber = allregmet[reg_idx][met_idx]
 
         # create data subset
-        spat_ludataharm_sub = spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)]
-        kernel_vector_sub = kernel_vector[(spat_region == regnumber) & (spat_met == metnumber)]
-        cons_data_sub = cons_data[(spat_region == regnumber) & (spat_met == metnumber)]
+        reg_met_mask = (spat_region == regnumber) & (spat_met == metnumber)
+        spat_ludataharm_sub = spat_ludataharm[reg_met_mask]
+        kernel_vector_sub = kernel_vector[reg_met_mask]
+        cons_data_sub = cons_data[reg_met_mask]
 
         # calculate intensification
         citz = _intensification(c.diagnostic, diag_file, spat_ludataharm_sub, target_intensification, kernel_vector_sub,
                                 cons_data_sub, reg_idx, metnumber, order_rules, final_landclasses, c.errortol,
                                 constrain_rules, target_change, transition_rules, land_mismatch)
 
         # apply intensification
-        spat_ludataharm[(spat_region == regnumber) & (spat_met == metnumber)], trans_mat, target_change, target_intensification = citz
+        spat_ludataharm[reg_met_mask], trans_mat, target_change, target_intensification = citz
 
         # log transition
-        transitions[(spat_region == regnumber) & (spat_met == metnumber), :, :] += trans_mat
+        transitions[reg_met_mask, :, :] += trans_mat
 
     # calculate non-achieved change
     non_chg = np.sum(abs(target_change[:, :, :])) / 2.

demeter/config_reader.py

@@ -16,6 +16,11 @@
 from configobj import ConfigObj
 
 
+class ValidationException(Exception):
+    def __init__(self,*args,**kwargs):
+        Exception.__init__(self,*args,**kwargs)
+
+
 class ReadConfig:
 
     def __init__(self, config_file):
@@ -191,14 +196,14 @@ def ck_ts(t, st_y, ed_y):
         ts = int(t)
 
         if (rng == 0) and (ts != 1):
-            raise RuntimeError('Parameter "timestep" value must be 1 if only running one year.  Your start year and end year are the same in your config file.  Exiting...')
+            raise ValidationException('Parameter "timestep" value must be 1 if only running one year.  Your start year and end year are the same in your config file.  Exiting...')
         elif (rng == 0) and (ts == 1):
             return ts
 
         ck = rng / ts
 
         if ck == 0:
-            raise RuntimeError('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}".  Max time step available based on year range is "{3}".  Exiting...'.format(t, st_y, ed_y, ed_y - st_y))
+            raise ValidationException('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}".  Max time step available based on year range is "{3}".  Exiting...'.format(t, st_y, ed_y, ed_y - st_y))
         else:
             return ts
 
@@ -212,7 +217,7 @@ def ck_yr(y, p):
         :return:            int
         """
         if len(y) != 4:
-            raise RuntimeError('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y))
+            raise ValidationException('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y))
         else:
             return int(y)
 
@@ -227,7 +232,7 @@ def ck_len(s, p, l=30):
         :return:            string
         """
         if len(s) > l:
-            raise RuntimeError('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
+            raise ValidationException('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
         else:
             return s
 
@@ -244,7 +249,7 @@ def ck_vals(v, p, l):
         if v in l:
             return v
         else:
-            raise RuntimeError('Value "{0}" not in acceptable values for parameter "{1}".  Acceptable values are:  {2}.  Exiting...'.format(v, p, l))
+            raise ValidationException('Value "{0}" not in acceptable values for parameter "{1}".  Acceptable values are:  {2}.  Exiting...'.format(v, p, l))
 
     @staticmethod
     def ck_limit(v, p, l):
@@ -259,7 +264,7 @@ def ck_limit(v, p, l):
         if (v >= l[0]) and (v <= l[1]):
             return v
         else:
-            raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
+            raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
 
     @staticmethod
     def check_exist(f, kind, log):
@@ -296,7 +301,7 @@ def create_dir(d, log):
         except Exception as e:
             log.error(e)
             log.error("ERROR:  Failed to create directory.")
-            sys.exit(1)
+            raise
 
     @staticmethod
     def ck_agg(a, log):
@@ -307,11 +312,11 @@ def ck_agg(a, log):
             agg = int(a)
         except TypeError:
             log.error('"agg_level" parameter must be either  1 or 2.  Exiting...')
-            sys.exit(1)
+            raise
 
         if agg < 1 or agg > 2:
             log.error('"agg_level" parameter must be either 1 or 2.  Exiting...')
-            sys.exit(1)
+            raise ValidationException
 
         else:
             return agg
@@ -433,7 +438,7 @@ def ck_limit(v, p, l):
         if (v >= l[0]) and (v <= l[1]):
             return v
         else:
-            raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
+            raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
 
     @staticmethod
     def ck_len(s, p, l=20):
@@ -446,7 +451,7 @@ def ck_len(s, p, l=20):
         :return:            string
         """
         if len(s) > l:
-            raise RuntimeError('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
+            raise ValidationException('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
         else:
             return s
 
@@ -696,14 +701,14 @@ def ck_ts(t, st_y, ed_y):
         ts = int(t)
 
         if (rng == 0) and (ts != 1):
-            raise RuntimeError('Parameter "timestep" value must be 1 if only running one year.  Your start year and end year are the same in your config file.  Exiting...')
+            raise ValidationException('Parameter "timestep" value must be 1 if only running one year.  Your start year and end year are the same in your config file.  Exiting...')
         elif (rng == 0) and (ts == 1):
             return ts
 
         ck = rng / ts
 
         if ck == 0:
-            raise RuntimeError('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}".  Max time step available based on year range is "{3}".  Exiting...'.format(t, st_y, ed_y, ed_y - st_y))
+            raise ValidationException('Parameter "timestep" value "{0}" is too large for start year of "{1}" and end year of "{2}".  Max time step available based on year range is "{3}".  Exiting...'.format(t, st_y, ed_y, ed_y - st_y))
         else:
             return ts
 
@@ -717,7 +722,7 @@ def ck_yr(y, p):
         :return:            int
         """
         if len(y) != 4:
-            raise RuntimeError('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y))
+            raise ValidationException('Year must be in four digit format (e.g., 2005) for parameter "{}". Value entered was "{}". Exiting...'.format(p, y))
         else:
             return int(y)
 
@@ -732,7 +737,7 @@ def ck_len(s, p, l=20):
         :return:            string
         """
         if len(s) > l:
-            raise RuntimeError('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
+            raise ValidationException('Length of "{}" exceeds the max length of 20.  Please revise.  Exiting...'.format(p))
         else:
             return s
 
@@ -749,7 +754,7 @@ def ck_vals(v, p, l):
         if v in l:
             return v
         else:
-            raise RuntimeError('Value "{0}" not in acceptable values for parameter "{1}".  Acceptable values are:  {2}.  Exiting...'.format(v, p, l))
+            raise ValidationException('Value "{0}" not in acceptable values for parameter "{1}".  Acceptable values are:  {2}.  Exiting...'.format(v, p, l))
 
     @staticmethod
     def ck_limit(v, p, l):
@@ -764,7 +769,7 @@ def ck_limit(v, p, l):
         if (v >= l[0]) and (v <= l[1]):
             return v
         else:
-            raise RuntimeError('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
+            raise ValidationException('Value "{0}" does not fall within acceptable range of values for parameter {1} where min >= {2} and max <= {3}. Exiting...'.format(v, p, l[0], l[1]))
 
 
     @staticmethod
@@ -779,11 +784,11 @@ def check_exist(f, kind, log):
         if kind == 'file' and os.path.isfile(f) is False:
             log.error("File not found:  {0}".format(f))
             log.error("Confirm path and retry.")
-            sys.exit()
+            raise IOError
         elif kind == 'dir' and os.path.isdir(f) is False:
             log.error("Directory not found:  {0}".format(f))
             log.error("Confirm path and retry.")
-            sys.exit()
+            raise IOError
         else:
             return f
 
@@ -802,7 +807,7 @@ def create_dir(d, log):
         except Exception as e:
             log.error(e)
             log.error("ERROR:  Failed to create directory.")
-            sys.exit()
+            raise ValidationException
 
     @staticmethod
     def ck_agg(a, log):
@@ -813,11 +818,11 @@ def ck_agg(a, log):
             agg = int(a)
         except TypeError:
             log.error('"agg_level" parameter must be either  1 or 2.  Exiting...')
-            sys.exit(1)
+            raise ValidationException
 
         if agg < 1 or agg > 2:
             log.error('"agg_level" parameter must be either 1 or 2.  Exiting...')
-            sys.exit(1)
+            raise ValidationException
         else:
             return agg
 
@@ -870,10 +875,4 @@ def get_constraints(self):
             return l
 
         else:
-            return list()
-
-if __name__ == "__main__":
-
-    ini = '/users/ladmin/repos/github/demeter/example/config.ini'
-
-    ReadConfigShuffle(ini, '/users/ladmin/Desktop')
+            return list()

demeter/constraints.py

@@ -9,11 +9,15 @@
 """
 import os
 import numpy as np
-import sys
 
 import demeter.demeter_io.reader as rdr
 
 
+class ValidationException(Exception):
+    def __init__(self,*args,**kwargs):
+        Exception.__init__(self,*args,**kwargs)
+
+
 class ApplyConstraints:
 
     def __init__(self, allreg, allaez, final_landclasses, user_years, ixr_ixm, allregaez, spat_region, allregnumber,
@@ -97,7 +101,7 @@ def apply_spat_constraints(self):
                 print("\nERROR: Aggregation numbers for PFT {0} in spatial allocation file sum up to more than 1.".format(i))
                 print("Please correct and try again.")
                 print("Exiting...\n")
-                sys.exit()
+                raise ValidationException
 
             # if individual values sum to greater than 1
             if np.sum(t > 0) > 1:
@@ -193,7 +197,7 @@ def apply_gcam_constraints(self, yr_idx, gcam_landmatrix, spat_landmatrix, ixr_i
                 print("\nERROR: No aggregation class defined for PFT {0} in the GCAM allocation file".format(self.gcam_landclasses(gix)))
                 print("Please correct and try again.")
                 print("Exiting...\n")
-                sys.exit()
+                raise ValidationException
 
             # Examine the case of one-to-many recombination (e.g., rockicedesert to snow and sparse). Data is split into
             #   the new categories following their share in the base land use layer within the considered region,

demeter/logger.py

@@ -86,5 +86,6 @@ def close_logger(self, log):
         """
         handlers = log.handlers[:]
         for h in handlers:
+            h.flush()
             h.close()
             log.removeHandler(h)

demeter/model.py

@@ -22,6 +22,11 @@
 from demeter.weight.kernel_density import KernelDensity
 
 
+class ValidationException(Exception):
+    def __init__(self,*args,**kwargs):
+        Exception.__init__(self,*args,**kwargs)
+
+
 class Demeter(Logger):
 
     def __init__(self,
@@ -301,7 +306,7 @@ def _shuffle(dir, i, oc):
                         2) the type of run you wish to conduct "standard" or "ensemble"
                     """)
         print('Exiting...')
-        sys.exit(1)
+        raise ValidationException
 
     # explode args
     ini = args[0]
@@ -313,12 +318,12 @@ def _shuffle(dir, i, oc):
         print('ERROR:  Config file not found.')
         print('You entered:  {0}'.format(ini))
         print('Please enter a full path file name with extension to config file and retry.')
-        sys.exit(1)
+        raise ValidationException
 
     if mode.lower() not in ('standard', 'ensemble'):
         print("""ERROR:  Run mode passed '{0}' not a valid option.  Either select 'standard' or 'ensemble' """.format(mode))
         print('Exiting...')
-        sys.exit(1)
+        raise ValidationException
 
     # instantiate demeter
     dm = Demeter(config=ini)