From 0166a3dc52fa2310cf7782d7d1cd9d8116615bf3 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Mon, 1 Dec 2025 10:30:01 +0100 Subject: [PATCH 01/23] Adapt path setting to match TiMBA --- c_module/parameters/paths.py | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 6e9b801..13964aa 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -14,20 +14,21 @@ def extract_scenarios(input_folder, output_folder, sc_num): :param sc_num: Number of scenarios to extract. :return: List of merged scenario names. """ - scenarios = [] + + folder_path = Path(output_folder) + files = list(folder_path.glob("*.pkl")) + files.sort(key=lambda f: f.stat().st_mtime) if sc_num is None: folder_path = Path(input_folder) - for file in folder_path.glob("*.xlsx"): - scenario_name = f"DataContainer_Sc_{file.stem}.pkl" - scenario_path = Path(output_folder) / Path(scenario_name) - scenarios.append(scenario_path) - + try: + sc_num = len(list(folder_path.glob("*.xlsx"))) + except FileNotFoundError: + sc_num = 1 + files = files[-sc_num:] else: - folder_path = Path(output_folder) - files = list(folder_path.glob("*.pkl")) - files.sort(key=lambda f: f.stat().st_mtime) files = files[-user_input[ParamNames.sc_num.value]:] - scenarios = files + + scenarios = files return scenarios @@ -58,10 +59,18 @@ def cmodule_is_standalone(): PACKAGEDIR = Path(__file__).parent.parent.absolute() -TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() -TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") -TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") +if user_input[ParamNames.folderpath.value] is None: + # If user-defined path does not exists, use default path + TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() + TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + +else: + # If user-defined path exist + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TIMBADIR_INPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("output") / Path("data") if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): # output paths for add-on c-module @@ -69,7 +78,10 @@ def cmodule_is_standalone(): else: # output paths for standalone c-module - OUTPUT_FOLDER = PACKAGEDIR / Path("data") / Path("output") + if user_input[ParamNames.folderpath.value] is None: + OUTPUT_FOLDER = PACKAGEDIR / Path("data") / Path("output") + else: + OUTPUT_FOLDER = USER_PATH / Path("data") / Path("output") # Official statistics from the Food and Agriculture Organization From 0ade70394272a59c0abeb3ac43658e2c70daa6c0 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Mon, 1 Dec 2025 10:31:06 +0100 Subject: [PATCH 02/23] Add user-defined folder path option --- c_module/cli/cli.py | 10 +++++++++- c_module/parameters/defines.py | 1 + c_module/user_io/default_parameters.py | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index cbab990..219e250 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -41,8 +41,14 @@ @click.option('-SD', '--show_carbon_dashboard', 'show_carbon_dashboard', default=user_input[ParamNames.show_carbon_dashboard.value], show_default=True, required=False, type=bool, help="Flag to launch carbon dashboard.") +@click.option('-UD', '--fao_data_update', 'fao_data_update', + default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, type=bool, + help="Flag to update FAO data.") +@click.option('-FP', '--folder_path', 'folder_path', default=user_input[ParamNames.folderpath.value], + show_default=True, required=False, type=str) def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_c_forest_bgb, calc_c_forest_soil, - calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard): + calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard, fao_data_update, + folder_path): user_input_cli = { ParamNames.add_on_activated.value: add_on_activated, @@ -57,6 +63,8 @@ def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_ ParamNames.calc_c_hwp.value: calc_c_hwp, ParamNames.c_hwp_accounting_approach.value: c_hwp_accounting_approach, ParamNames.show_carbon_dashboard.value: show_carbon_dashboard, + ParamNames.fao_data_update.value: fao_data_update, + ParamNames.folderpath.value: folder_path, # Adavanced settings not available via CLI ParamNames.historical_c_hwp.value: user_input[ParamNames.historical_c_hwp.value], ParamNames.hist_hwp_start_year.value: user_input[ParamNames.hist_hwp_start_year.value], diff --git a/c_module/parameters/defines.py b/c_module/parameters/defines.py index 9bbb764..ccc7499 100644 --- a/c_module/parameters/defines.py +++ b/c_module/parameters/defines.py @@ -28,6 +28,7 @@ class ParamNames(Enum): start_year = "start_year" end_year = "end_year" read_in_pkl = "read_in_pkl" + folderpath = "folderpath" save_data_as = "save_data_as" calc_c_forest_agb = "calc_c_forest_agb" calc_c_forest_bgb = "calc_c_forest_bgb" diff --git a/c_module/user_io/default_parameters.py b/c_module/user_io/default_parameters.py index 4e25614..5a3c309 100644 --- a/c_module/user_io/default_parameters.py +++ b/c_module/user_io/default_parameters.py @@ -9,6 +9,7 @@ end_year = 2050 # Not activated read_in_pkl = True # Caution False option is not implemented yet +folderpath = None # Forest carbon related parameters calc_c_forest_agb = True @@ -35,6 +36,7 @@ ParamNames.start_year.value: start_year, ParamNames.end_year.value: end_year, ParamNames.read_in_pkl.value: read_in_pkl, + ParamNames.folderpath.value: folderpath, ParamNames.calc_c_forest_agb.value: calc_c_forest_agb, ParamNames.calc_c_forest_bgb.value: calc_c_forest_bgb, ParamNames.calc_c_forest_soil.value: calc_c_forest_soil, From e8e8f4670f351a081d1a58bf769ce45dda249aa8 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Mon, 1 Dec 2025 10:31:45 +0100 Subject: [PATCH 03/23] Adapt output file naming --- c_module/data_management/data_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 11a1ab5..214bc21 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -109,12 +109,11 @@ def save_data(self): if not self.UserInput[ParamNames.add_on_activated.value]: DataManager.serialize_to_pickle(self.timba_data[sc], OUTPUT_FOLDER / Path(f"{sc}.pkl")) else: - DataManager.serialize_to_pickle( - self.carbon_data[sc], OUTPUT_FOLDER / Path(f"{self.time_stamp}_{sc}.pkl")) + DataManager.serialize_to_pickle(self.carbon_data[sc], OUTPUT_FOLDER / Path(f"{sc}.pkl")) for df_key in self.carbon_data[sc].keys(): carbon_data = self.carbon_data[sc][df_key] - carbon_data_path = OUTPUT_FOLDER / Path(f"{df_key}_D{self.time_stamp}_{sc}") + carbon_data_path = OUTPUT_FOLDER / Path(f"{df_key}_{sc}") carbon_data.to_csv(f"{carbon_data_path}.csv", index=False) @staticmethod From 99104caad344b425aedd6f0f3b210d1d973eadad Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Mon, 1 Dec 2025 11:01:06 +0100 Subject: [PATCH 04/23] Add help text for cli --- c_module/cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index 219e250..c9cdd36 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -45,7 +45,7 @@ default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, type=bool, help="Flag to update FAO data.") @click.option('-FP', '--folder_path', 'folder_path', default=user_input[ParamNames.folderpath.value], - show_default=True, required=False, type=str) + show_default=True, required=False, type=str, help="Path to directory with Input/Output folder.") def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_c_forest_bgb, calc_c_forest_soil, calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard, fao_data_update, folder_path): From 04f242cbba3d48a245f540e176064ed58c596120 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Mon, 1 Dec 2025 17:35:26 +0100 Subject: [PATCH 05/23] add debug steps for cli --- c_module/data_management/data_manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 214bc21..b823cbb 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -20,7 +20,9 @@ class DataManager: def set_sc_paths(self): if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): # input paths for add-on c-module - + print(TIMBADIR_INPUT) + print(TIMBADIR_INPUT) + print(TIMBADIR_INPUT) scenarios = extract_scenarios(input_folder=TIMBADIR_INPUT, output_folder=TIMBADIR_OUTPUT, sc_num=user_input[ParamNames.sc_num.value]) @@ -138,7 +140,9 @@ def retrieve_commodity_num(self): commodity_dict = VarNames.commodity_dict.value commodity_code = VarNames.commodity_code.value commodity_num_name = VarNames.commodity_num.value - + print(self.sc_list[0]) + print(self.sc_list[0]) + print(self.sc_list[0]) commodity_num = len(self.timba_data[self.sc_list[0]][timba_data_all][commodity_code].unique()) self.add_data[commodity_dict] = {} self.add_data[commodity_dict][commodity_num_name] = commodity_num From 25189098879fda71595d77f1faf3bfadad4d8b7c Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 08:05:55 +0100 Subject: [PATCH 06/23] add debug steps for cli --- c_module/data_management/data_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index b823cbb..13ad462 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -20,6 +20,10 @@ class DataManager: def set_sc_paths(self): if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): # input paths for add-on c-module + print("\n") + print(f"carbon module is standalone: {cmodule_is_standalone()}") + print(f"user input add-on: {user_input[ParamNames.add_on_activated.value]}") + print("\n") print(TIMBADIR_INPUT) print(TIMBADIR_INPUT) print(TIMBADIR_INPUT) From a9cc61cd72dde74eac4d774be95db91e05f49326 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 13:24:49 +0100 Subject: [PATCH 07/23] Reset debug steps for cli --- c_module/data_management/data_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 13ad462..e3aaaf0 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -20,13 +20,6 @@ class DataManager: def set_sc_paths(self): if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): # input paths for add-on c-module - print("\n") - print(f"carbon module is standalone: {cmodule_is_standalone()}") - print(f"user input add-on: {user_input[ParamNames.add_on_activated.value]}") - print("\n") - print(TIMBADIR_INPUT) - print(TIMBADIR_INPUT) - print(TIMBADIR_INPUT) scenarios = extract_scenarios(input_folder=TIMBADIR_INPUT, output_folder=TIMBADIR_OUTPUT, sc_num=user_input[ParamNames.sc_num.value]) @@ -144,9 +137,6 @@ def retrieve_commodity_num(self): commodity_dict = VarNames.commodity_dict.value commodity_code = VarNames.commodity_code.value commodity_num_name = VarNames.commodity_num.value - print(self.sc_list[0]) - print(self.sc_list[0]) - print(self.sc_list[0]) commodity_num = len(self.timba_data[self.sc_list[0]][timba_data_all][commodity_code].unique()) self.add_data[commodity_dict] = {} self.add_data[commodity_dict][commodity_num_name] = commodity_num From c8c04ecdb22162f900dc73cd26ec83d6f03de6c5 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 13:26:01 +0100 Subject: [PATCH 08/23] Extend cmodule_is_standalone function --- c_module/data_management/data_manager.py | 2 +- c_module/parameters/paths.py | 85 ++++++++++++++++++++---- 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index e3aaaf0..1896856 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -18,7 +18,7 @@ class DataManager: @staticmethod def set_sc_paths(self): - if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): # input paths for add-on c-module scenarios = extract_scenarios(input_folder=TIMBADIR_INPUT, output_folder=TIMBADIR_OUTPUT, diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 13964aa..20958bc 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -14,7 +14,6 @@ def extract_scenarios(input_folder, output_folder, sc_num): :param sc_num: Number of scenarios to extract. :return: List of merged scenario names. """ - folder_path = Path(output_folder) files = list(folder_path.glob("*.pkl")) files.sort(key=lambda f: f.stat().st_mtime) @@ -33,28 +32,86 @@ def extract_scenarios(input_folder, output_folder, sc_num): return scenarios -def cmodule_is_standalone(): +def cmodule_is_standalone(debug: bool = False) -> bool: """ Check if cmodule is standalone or not, covering if the code is run as the main program, covering CLI, script, IDE, and entry point runs. + :param debug: Flag to enable debug mode. :return: Bool if cmodule is standalone or not. """ - import __main__ import sys + import inspect + import __main__ - if getattr(__main__, "__file__", None): - main_file = Path(__main__.__file__).resolve() - package_root = Path(__file__).resolve().parents[1] - - if package_root in main_file.parents: - return True - - if "pytest" in sys.modules and Path.cwd().resolve() == package_root.parent: - return True - - if any("unittest" in mod for mod in sys.modules): + reasons = [] + + # Running under a typical test runner => treat as imported + if "pytest" in sys.modules: + reasons.append("pytest detected in sys.modules") + if debug: + print("DEBUG: pytest present -> treated as imported") + return False + if any("unittest" in mod for mod in sys.modules): + reasons.append("unittest detected in sys.modules") + if debug: + print("DEBUG: unittest present -> treated as imported") + return False + + # Simple and reliable check for most cases + if __name__ == "__main__": + reasons.append("__name__ == '__main__'") + if debug: + print("DEBUG: __name__ == '__main__' -> standalone") + return True + + # Inspect stack: some IDEs or runners execute a wrapper that sets __name__ == '__main__' + # in a different frame. If any frame was executed as __main__, assume standalone entry. + for frame_info in inspect.stack(): + g = frame_info.frame.f_globals + frame_name = g.get("__name__") + frame_file = g.get("__file__", None) + if frame_name == "__main__": + reasons.append(f"found frame with __name__ == '__main__' (file={frame_file})") + if debug: + print("DEBUG: stack frame with __name__ == '__main__' -> standalone") + print(f"DEBUG: frame file: {frame_file}") return True + # Compare the top-level script path with this package path: + # if the top-level entry script is outside this package, it likely invoked/imported the package. + main_file = getattr(__main__, "__file__", None) + if main_file: + try: + main_path = Path(main_file).resolve() + package_root = Path(__file__).resolve().parents[1] + # If the top-level script is the module file itself -> standalone + if main_path == Path(__file__).resolve(): + reasons.append("main_file equals this module file") + if debug: + print("DEBUG: main_file equals this module file -> standalone") + return True + # If the top-level script is *inside* the package: often still a standalone run (python -m) + if package_root in main_path.parents: + reasons.append("main_file is located inside package root (likely -m or IDE module run)") + if debug: + print("DEBUG: main_file inside package root -> standalone") + print(f"DEBUG: main_file={main_path}, package_root={package_root}") + return True + # Otherwise treat as imported + reasons.append("main_file exists but is outside package -> treated as imported") + if debug: + print("DEBUG: main_file outside package -> treated as imported") + print(f"DEBUG: main_file={main_path}, package_root={package_root}") + return False + except Exception as e: + # fallback + if debug: + print("DEBUG: error resolving main_file or package_root:", e) + pass + + # If none of the above matched, assume imported + if debug: + print("DEBUG: no indication of standalone execution; reasons:", reasons) return False From b46465772e313ba61b3f8704244d4678122c6976 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 13:26:38 +0100 Subject: [PATCH 09/23] Refine path settings --- c_module/parameters/paths.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 20958bc..77bb124 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -116,30 +116,36 @@ def cmodule_is_standalone(debug: bool = False) -> bool: PACKAGEDIR = Path(__file__).parent.parent.absolute() -INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") -if user_input[ParamNames.folderpath.value] is None: - # If user-defined path does not exists, use default path - TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() - TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") - TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") -else: - # If user-defined path exist - USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() - TIMBADIR_INPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") - TIMBADIR_OUTPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("output") / Path("data") +if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): + # input and output paths for add-on c-module + if user_input[ParamNames.folderpath.value] is None: + # If user-defined path does not exists, use default path + # For compatibility with other modules, paths must be adapted + TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() + TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + else: + # If user-defined path exist + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TIMBADIR_INPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("output") / Path("data") -if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): - # output paths for add-on c-module OUTPUT_FOLDER = TIMBADIR_OUTPUT else: - # output paths for standalone c-module + # input and output paths for standalone c-module if user_input[ParamNames.folderpath.value] is None: + INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") OUTPUT_FOLDER = PACKAGEDIR / Path("data") / Path("output") else: + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + INPUT_FOLDER = USER_PATH / Path("data") / Path("input") OUTPUT_FOLDER = USER_PATH / Path("data") / Path("output") + TIMBADIR_INPUT = None + TIMBADIR_OUTPUT = None + # Official statistics from the Food and Agriculture Organization FAO_DIR = INPUT_FOLDER / Path("historical_data") From 852f93f648e558e7572f88212b4bbbef904368e2 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 13:54:41 +0100 Subject: [PATCH 10/23] Add safeguard for inconsistent settings --- c_module/parameters/paths.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 77bb124..baaea45 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -117,6 +117,15 @@ def cmodule_is_standalone(debug: bool = False) -> bool: PACKAGEDIR = Path(__file__).parent.parent.absolute() +if cmodule_is_standalone(debug=False): + if user_input[ParamNames.add_on_activated.value]: + import sys + print("Inconsistent settings:") + print(f"C-Module is executed as standalone: {cmodule_is_standalone(debug=False)}") + print(f"But parameter add_on_activated: {user_input[ParamNames.add_on_activated.value]}") + print(f"Harmonize settings to proceed") + sys.exit("Stopping execution.") + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): # input and output paths for add-on c-module if user_input[ParamNames.folderpath.value] is None: From c3e790c9ecdad5dd63858a26344e03a2a604411f Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 13:55:03 +0100 Subject: [PATCH 11/23] Simplify path setting --- c_module/parameters/paths.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index baaea45..1536cab 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -132,29 +132,32 @@ def cmodule_is_standalone(debug: bool = False) -> bool: # If user-defined path does not exists, use default path # For compatibility with other modules, paths must be adapted TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() - TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") - TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + TARGETDIR = TIMBADIR else: # If user-defined path exist USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() - TIMBADIR_INPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") - TIMBADIR_OUTPUT = USER_PATH / Path("TiMBA") / Path("data") / Path("output") / Path("data") + TARGETDIR = USER_PATH + TIMBADIR_INPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + + INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") OUTPUT_FOLDER = TIMBADIR_OUTPUT else: # input and output paths for standalone c-module if user_input[ParamNames.folderpath.value] is None: - INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") - OUTPUT_FOLDER = PACKAGEDIR / Path("data") / Path("output") + TARGETDIR = PACKAGEDIR else: USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() - INPUT_FOLDER = USER_PATH / Path("data") / Path("input") - OUTPUT_FOLDER = USER_PATH / Path("data") / Path("output") + TARGETDIR = USER_PATH TIMBADIR_INPUT = None TIMBADIR_OUTPUT = None + INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") + OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") + # Official statistics from the Food and Agriculture Organization FAO_DIR = INPUT_FOLDER / Path("historical_data") From 4e2854f90581d7d7526f8f2e5e8cb6d5578c5eca Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Tue, 2 Dec 2025 16:00:50 +0100 Subject: [PATCH 12/23] Add data input checks --- c_module/data_management/data_manager.py | 80 ++++++++++++++++++++- c_module/data_management/process_manager.py | 1 + c_module/parameters/defines.py | 8 +++ c_module/parameters/paths.py | 2 + 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 1896856..19260f8 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -1,8 +1,8 @@ from c_module.parameters.paths import (INPUT_FOLDER, TIMBADIR_INPUT, ADD_INFO_CARBON_PATH, ADD_INFO_COUNTRY, FAOSTAT_DATA, FRA_DATA, OUTPUT_FOLDER, TIMBADIR_OUTPUT, FAOSTAT_URL, FAO_DIR, - FRA_URL) + FRA_URL, DEFAULT_PROJECTION_URL, ADD_INFO_URL) from c_module.parameters.paths import cmodule_is_standalone, extract_scenarios -from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants) +from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants, FolderNames) from c_module.user_io.default_parameters import user_input import pandas as pd from tqdm import tqdm @@ -34,6 +34,82 @@ def set_sc_paths(self): self.sc_path = PKL_RESULTS_INPUT + @staticmethod + def check_input_data(self): + self.logger.info(f"C-Module - Check input data for carbon module") + DataManager.check_input_data_structure(self) + DataManager.check_input_data_content(self) + + @staticmethod + def check_input_data_structure(self): + INPUT_FOLDER.mkdir(parents=True, exist_ok=True) + + if cmodule_is_standalone(debug=False): + required = {FolderNames.additional_info.value, FolderNames.projection_data.value} + else: + required = {FolderNames.additional_info.value} + existing = {p.name for p in Path(INPUT_FOLDER).iterdir() if p.is_dir()} + missing = list(required - existing) + if len(missing) > 0: + for missing_folder in missing: + NEW_FOLDER = INPUT_FOLDER / Path(missing_folder) + NEW_FOLDER.mkdir(parents=True, exist_ok=True) + + @staticmethod + def check_input_data_content(self): + subfolders = [p.name for p in INPUT_FOLDER.iterdir() if p.is_dir()] + for folder in subfolders: + if (folder == FolderNames.additional_info.value) or (folder == FolderNames.projection_data.value): + if folder == FolderNames.additional_info.value: + # download additional info data + GIT_DATA_URL = ADD_INFO_URL + + if folder == FolderNames.projection_data.value: + # download projection data + GIT_DATA_URL = DEFAULT_PROJECTION_URL + + folder_path = INPUT_FOLDER / Path(folder) + missing_files = DataManager.compare_local_and_remote(local_folder_path=folder_path, + remote_folder_url=GIT_DATA_URL) + + for missing_file in list(missing_files): + DataManager.download_carbon_data_from_github(self=self, + data_url=GIT_DATA_URL, + folder_path=folder_path, + missing_file=missing_file) + + @staticmethod + def compare_local_and_remote(local_folder_path, remote_folder_url): + response = requests.get(remote_folder_url, timeout=30) + response.raise_for_status() + repo_files = response.json() + + github_filenames = {f["name"] for f in repo_files if f["type"] == "file"} + + local_filenames = {p.name for p in local_folder_path.iterdir() if p.is_file()} + + missing_local = github_filenames - local_filenames + + return missing_local + + @staticmethod + def download_carbon_data_from_github(self, data_url, folder_path, missing_file): + response = requests.get(data_url, timeout=30) + response.raise_for_status() + + files = response.json() + for file in files: + if file["name"] == missing_file: + self.logger.info(f"C-Module - Download {file['name']} from GitHub") + if file["type"] == "file": + r = requests.get(file["download_url"], timeout=30) + r.raise_for_status() + out_file = folder_path / file["name"] + + with open(out_file, "wb") as f: + f.write(r.content) + + @staticmethod def load_data(filepath, table_name, input_source): if input_source.lower() == "excel": diff --git a/c_module/data_management/process_manager.py b/c_module/data_management/process_manager.py index 65357e6..e1117da 100644 --- a/c_module/data_management/process_manager.py +++ b/c_module/data_management/process_manager.py @@ -8,6 +8,7 @@ class ProcessManager: @staticmethod def run_readin_process(self): + DataManager.check_input_data(self) DataManager.set_sc_paths(self) ProcessManager.readin_add_data_process(self) ProcessManager.readin_timba_process(self) diff --git a/c_module/parameters/defines.py b/c_module/parameters/defines.py index ccc7499..acddc29 100644 --- a/c_module/parameters/defines.py +++ b/c_module/parameters/defines.py @@ -138,6 +138,14 @@ class VarNames(Enum): faostat_export_value = "Export value" +class FolderNames(Enum): + # Folder names + additional_info = "additional_information" + projection_data = "projection_data" + historical_data = "historical_data" + + + diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 1536cab..0e5e9e6 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -167,10 +167,12 @@ def cmodule_is_standalone(debug: bool = False) -> bool: FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") # additional information +ADD_INFO_URL = "https://api.github.com/repos/TI-Forest-Sector-Modelling/C-Module/contents/c_module/data/input/additional_information" ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") +DEFAULT_PROJECTION_URL = "https://api.github.com/repos/TI-Forest-Sector-Modelling/C-Module/contents/c_module/data/input/projection_data" LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER From 898598f51ab1a36dbc6097efd2f91b512cfab105 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 13:37:09 +0100 Subject: [PATCH 13/23] GitHub access without API --- c_module/data_management/data_manager.py | 93 ++++++++++++++++++------ c_module/parameters/paths.py | 5 +- 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 19260f8..525dac5 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -1,6 +1,6 @@ from c_module.parameters.paths import (INPUT_FOLDER, TIMBADIR_INPUT, ADD_INFO_CARBON_PATH, ADD_INFO_COUNTRY, FAOSTAT_DATA, FRA_DATA, OUTPUT_FOLDER, TIMBADIR_OUTPUT, FAOSTAT_URL, FAO_DIR, - FRA_URL, DEFAULT_PROJECTION_URL, ADD_INFO_URL) + FRA_URL, DEFAULT_PROJECTION_DIR, ADD_INFO_DIR, CMODULE_ZIP_URL) from c_module.parameters.paths import cmodule_is_standalone, extract_scenarios from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants, FolderNames) from c_module.user_io.default_parameters import user_input @@ -8,6 +8,7 @@ from tqdm import tqdm from pathlib import Path import requests +from io import BytesIO import zipfile import io import time @@ -36,12 +37,19 @@ def set_sc_paths(self): @staticmethod def check_input_data(self): + """ + Checks input data for the C-Module in two steps. First, the input data structure is checked. After, the content + of each input data folder is checked. + """ self.logger.info(f"C-Module - Check input data for carbon module") DataManager.check_input_data_structure(self) DataManager.check_input_data_content(self) @staticmethod def check_input_data_structure(self): + """ + Checks the input data structure. If input data folder are missing, the missing folder is generated. + """ INPUT_FOLDER.mkdir(parents=True, exist_ok=True) if cmodule_is_standalone(debug=False): @@ -57,58 +65,99 @@ def check_input_data_structure(self): @staticmethod def check_input_data_content(self): + """ + Checks if input data folder content corresponds to folder content from the C-Module main branch on GitHub. + Missing input data is downloaded automatically. + :param self: C-Module object + """ subfolders = [p.name for p in INPUT_FOLDER.iterdir() if p.is_dir()] for folder in subfolders: if (folder == FolderNames.additional_info.value) or (folder == FolderNames.projection_data.value): if folder == FolderNames.additional_info.value: # download additional info data - GIT_DATA_URL = ADD_INFO_URL + GIT_DATA_DIR = ADD_INFO_DIR if folder == FolderNames.projection_data.value: # download projection data - GIT_DATA_URL = DEFAULT_PROJECTION_URL + GIT_DATA_DIR = DEFAULT_PROJECTION_DIR folder_path = INPUT_FOLDER / Path(folder) missing_files = DataManager.compare_local_and_remote(local_folder_path=folder_path, - remote_folder_url=GIT_DATA_URL) + repo_zip_url=CMODULE_ZIP_URL, + target_subdir=GIT_DATA_DIR) for missing_file in list(missing_files): DataManager.download_carbon_data_from_github(self=self, - data_url=GIT_DATA_URL, + repo_zip_url=CMODULE_ZIP_URL, + target_subdir=GIT_DATA_DIR, folder_path=folder_path, missing_file=missing_file) @staticmethod - def compare_local_and_remote(local_folder_path, remote_folder_url): - response = requests.get(remote_folder_url, timeout=30) + def compare_local_and_remote(local_folder_path: Path, repo_zip_url: str, target_subdir: str): + """ + Compares local and remote input data folder and returns missing files. + :param local_folder_path: Local input data folder + :param repo_zip_url: Remote input data zip url + :param target_subdir: Target subdirectory of remote input data folder + :return: Missing files in local folder + """ + response = requests.get(repo_zip_url, timeout=60) response.raise_for_status() - repo_files = response.json() - github_filenames = {f["name"] for f in repo_files if f["type"] == "file"} + with zipfile.ZipFile(BytesIO(response.content)) as zip_file: + zip_files = zip_file.namelist() + + # GitHub files + github_filenames = { + Path(f).name + for f in zip_files + if f.startswith(target_subdir) and not f.endswith("/") + } - local_filenames = {p.name for p in local_folder_path.iterdir() if p.is_file()} + # Local files + local_filenames = { + p.name for p in local_folder_path.iterdir() if p.is_file() + } missing_local = github_filenames - local_filenames return missing_local @staticmethod - def download_carbon_data_from_github(self, data_url, folder_path, missing_file): - response = requests.get(data_url, timeout=30) + def download_carbon_data_from_github(self, repo_zip_url: str, target_subdir: str, folder_path: Path, + missing_file: str): + """ + Downloads missing input data from GitHub. + :param self: C-Module object + :param repo_zip_url: Remote input data zip url + :param target_subdir: Target subdirectory of remote input data folder + :param folder_path: Local input data folder + :param missing_file: Input data files missing in local folder + """ + response = requests.get(repo_zip_url, timeout=60) response.raise_for_status() - files = response.json() - for file in files: - if file["name"] == missing_file: - self.logger.info(f"C-Module - Download {file['name']} from GitHub") - if file["type"] == "file": - r = requests.get(file["download_url"], timeout=30) - r.raise_for_status() - out_file = folder_path / file["name"] + with zipfile.ZipFile(BytesIO(response.content)) as zip_file: + zip_files = zip_file.namelist() + + target_path = None + for f in zip_files: + if f.startswith(target_subdir) and f.endswith(missing_file): + target_path = f + break + + if not target_path: + raise FileNotFoundError( + f"{missing_file} not found in GitHub folder {target_subdir}" + ) - with open(out_file, "wb") as f: - f.write(r.content) + self.logger.info(f"C-Module - Download {missing_file} from GitHub") + with zip_file.open(target_path) as zf: + out_file = folder_path / missing_file + with open(out_file, "wb") as f: + f.write(zf.read()) @staticmethod def load_data(filepath, table_name, input_source): diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 0e5e9e6..e712128 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -167,12 +167,13 @@ def cmodule_is_standalone(debug: bool = False) -> bool: FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") # additional information -ADD_INFO_URL = "https://api.github.com/repos/TI-Forest-Sector-Modelling/C-Module/contents/c_module/data/input/additional_information" +CMODULE_ZIP_URL = "https://github.com/TI-Forest-Sector-Modelling/C-Module/archive/refs/heads/main.zip" +ADD_INFO_DIR = "C-Module-main/c_module/data/input/additional_information" ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") -DEFAULT_PROJECTION_URL = "https://api.github.com/repos/TI-Forest-Sector-Modelling/C-Module/contents/c_module/data/input/projection_data" +DEFAULT_PROJECTION_DIR = "C-Module-main/c_module/data/input/projection_data" LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER From b2c1e5d505cf8826b9d00116f5dc2b9695776e89 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 14:34:29 +0100 Subject: [PATCH 14/23] Activate debug trigger --- c_module/parameters/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index e712128..c676ef8 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -126,7 +126,7 @@ def cmodule_is_standalone(debug: bool = False) -> bool: print(f"Harmonize settings to proceed") sys.exit("Stopping execution.") -if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): +if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=True): # input and output paths for add-on c-module if user_input[ParamNames.folderpath.value] is None: # If user-defined path does not exists, use default path From 051bf1e47d5ac006da13e9a4425dede4e91f9c5a Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 14:52:36 +0100 Subject: [PATCH 15/23] Add path print for debug --- c_module/parameters/paths.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index c676ef8..390edcc 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -158,6 +158,12 @@ def cmodule_is_standalone(debug: bool = False) -> bool: INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") +print("Input path used") +print(INPUT_FOLDER) +print("\n") +print("Output path used") +print(OUTPUT_FOLDER) + # Official statistics from the Food and Agriculture Organization FAO_DIR = INPUT_FOLDER / Path("historical_data") From 777206fadb3e9597df0289a689be87db0252ffb7 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 14:59:08 +0100 Subject: [PATCH 16/23] Add path print for debug --- c_module/parameters/paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 390edcc..a4d3187 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -151,7 +151,7 @@ def cmodule_is_standalone(debug: bool = False) -> bool: else: USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() TARGETDIR = USER_PATH - + print(f"USER PATH used: {USER_PATH}") TIMBADIR_INPUT = None TIMBADIR_OUTPUT = None From 1e9bc0064907a21fd67abe649c281c704ceb2d41 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 15:04:10 +0100 Subject: [PATCH 17/23] Test folderpath --- c_module/user_io/default_parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_module/user_io/default_parameters.py b/c_module/user_io/default_parameters.py index 5a3c309..711ee3e 100644 --- a/c_module/user_io/default_parameters.py +++ b/c_module/user_io/default_parameters.py @@ -9,7 +9,7 @@ end_year = 2050 # Not activated read_in_pkl = True # Caution False option is not implemented yet -folderpath = None +folderpath = "C:/Users/honkomp/test_test_test" # Forest carbon related parameters calc_c_forest_agb = True From af30fe7db1c50bfd98b629e4f1aa373ec199c5f9 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 15:24:05 +0100 Subject: [PATCH 18/23] Harmonize folderpath parameter --- c_module/cli/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index c9cdd36..4d22793 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -44,11 +44,11 @@ @click.option('-UD', '--fao_data_update', 'fao_data_update', default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, type=bool, help="Flag to update FAO data.") -@click.option('-FP', '--folder_path', 'folder_path', default=user_input[ParamNames.folderpath.value], +@click.option('-FP', '--folderpath', 'folderpath', default=user_input[ParamNames.folderpath.value], show_default=True, required=False, type=str, help="Path to directory with Input/Output folder.") def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_c_forest_bgb, calc_c_forest_soil, calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard, fao_data_update, - folder_path): + folderpath): user_input_cli = { ParamNames.add_on_activated.value: add_on_activated, @@ -64,7 +64,7 @@ def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_ ParamNames.c_hwp_accounting_approach.value: c_hwp_accounting_approach, ParamNames.show_carbon_dashboard.value: show_carbon_dashboard, ParamNames.fao_data_update.value: fao_data_update, - ParamNames.folderpath.value: folder_path, + ParamNames.folderpath.value: folderpath, # Adavanced settings not available via CLI ParamNames.historical_c_hwp.value: user_input[ParamNames.historical_c_hwp.value], ParamNames.hist_hwp_start_year.value: user_input[ParamNames.hist_hwp_start_year.value], From 7b2a3053238339ab83aea6298788e043ea4c3917 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 15:34:27 +0100 Subject: [PATCH 19/23] Adapt CLI for bool options --- c_module/cli/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index 4d22793..28089cd 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -6,7 +6,7 @@ @click.command() @click.option('-ADD_ON', '--add_on_activated', "add_on_activated", - default=user_input[ParamNames.add_on_activated.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.add_on_activated.value], show_default=True, required=True, is_flag=True, help="Flag to use the carbon module as a standalone module or as a TiMBA add-on.") @click.option('-SC', '--sc_num', "sc_num", default=user_input[ParamNames.sc_num.value], show_default=True, required=True, type=int, @@ -18,31 +18,31 @@ show_default=True, required=True, type=int, help="End year of carbon calculations.") @click.option('-CF_AGB', '--calc_c_forest_agb', "calc_c_forest_agb", - default=user_input[ParamNames.calc_c_forest_agb.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_agb.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for aboveground forest biomass.") @click.option('-CF_BGB', '--calc_c_forest_bgb', "calc_c_forest_bgb", - default=user_input[ParamNames.calc_c_forest_bgb.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_bgb.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for belowground forest biomass.") @click.option('-CF_S', '--calc_c_forest_soil', "calc_c_forest_soil", - default=user_input[ParamNames.calc_c_forest_soil.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_soil.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for forest soil.") @click.option('-CF_DWL', '--calc_c_forest_dwl', "calc_c_forest_dwl", - default=user_input[ParamNames.calc_c_forest_dwl.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_dwl.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for dead wood and litter.") @click.option('-C_HWP', '--calc_c_hwp', "calc_c_hwp", - default=user_input[ParamNames.calc_c_hwp.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_hwp.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for harvested wood products.") @click.option('-C_HWP_A', '--c_hwp_accounting_approach', "c_hwp_accounting_approach", default=user_input[ParamNames.c_hwp_accounting_approach.value], show_default=True, required=True, type=str, help="Flag to select the accounting approach for carbon in harvested wood products.") @click.option('-R', '--read_in_pkl', "read_in_pkl", - default=user_input[ParamNames.read_in_pkl.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.read_in_pkl.value], show_default=True, required=True, is_flag=True, help="Flag to control if pkl- or csv-files are read; reads in if True.") @click.option('-SD', '--show_carbon_dashboard', 'show_carbon_dashboard', - default=user_input[ParamNames.show_carbon_dashboard.value], show_default=True, required=False, type=bool, - help="Flag to launch carbon dashboard.") + default=user_input[ParamNames.show_carbon_dashboard.value], show_default=True, required=False, + is_flag=True, help="Flag to launch carbon dashboard.") @click.option('-UD', '--fao_data_update', 'fao_data_update', - default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, type=bool, + default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, is_flag=True, help="Flag to update FAO data.") @click.option('-FP', '--folderpath', 'folderpath', default=user_input[ParamNames.folderpath.value], show_default=True, required=False, type=str, help="Path to directory with Input/Output folder.") From c83bb0c2ebfa204247b957237c9d3de35e28886c Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 15:49:33 +0100 Subject: [PATCH 20/23] Adapt CLI for bool options --- c_module/cli/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index 28089cd..519dad9 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -46,6 +46,7 @@ help="Flag to update FAO data.") @click.option('-FP', '--folderpath', 'folderpath', default=user_input[ParamNames.folderpath.value], show_default=True, required=False, type=str, help="Path to directory with Input/Output folder.") + def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_c_forest_bgb, calc_c_forest_soil, calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard, fao_data_update, folderpath): From aafa6ce326ef2347908c2b7bddee2df8de307cb0 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 17:24:50 +0100 Subject: [PATCH 21/23] Implement full dynamic paths --- c_module/data_management/data_manager.py | 28 +++- c_module/data_management/process_manager.py | 5 +- c_module/logic/base_logger.py | 5 +- c_module/logic/carbon_calc.py | 16 ++- c_module/logic/main.py | 7 +- c_module/parameters/defines.py | 22 +++ c_module/parameters/paths.py | 150 +++++++++++--------- c_module/user_io/default_parameters.py | 2 +- 8 files changed, 150 insertions(+), 85 deletions(-) diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 525dac5..636a6a6 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -1,8 +1,5 @@ -from c_module.parameters.paths import (INPUT_FOLDER, TIMBADIR_INPUT, ADD_INFO_CARBON_PATH, ADD_INFO_COUNTRY, - FAOSTAT_DATA, FRA_DATA, OUTPUT_FOLDER, TIMBADIR_OUTPUT, FAOSTAT_URL, FAO_DIR, - FRA_URL, DEFAULT_PROJECTION_DIR, ADD_INFO_DIR, CMODULE_ZIP_URL) from c_module.parameters.paths import cmodule_is_standalone, extract_scenarios -from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants, FolderNames) +from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants, FolderNames, PathNames) from c_module.user_io.default_parameters import user_input import pandas as pd from tqdm import tqdm @@ -19,6 +16,10 @@ class DataManager: @staticmethod def set_sc_paths(self): + TIMBADIR_INPUT = self.paths[PathNames.TIMBADIR_INPUT.value] + TIMBADIR_OUTPUT = self.paths[PathNames.TIMBADIR_OUTPUT.value] + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): # input paths for add-on c-module scenarios = extract_scenarios(input_folder=TIMBADIR_INPUT, @@ -50,6 +51,7 @@ def check_input_data_structure(self): """ Checks the input data structure. If input data folder are missing, the missing folder is generated. """ + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] INPUT_FOLDER.mkdir(parents=True, exist_ok=True) if cmodule_is_standalone(debug=False): @@ -70,6 +72,11 @@ def check_input_data_content(self): Missing input data is downloaded automatically. :param self: C-Module object """ + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] + CMODULE_ZIP_URL = self.paths[PathNames.CMODULE_ZIP_URL.value] + ADD_INFO_DIR = self.paths[PathNames.ADD_INFO_DIR.value] + DEFAULT_PROJECTION_DIR = self.paths[PathNames.DEFAULT_PROJECTION_DIR.value] + subfolders = [p.name for p in INPUT_FOLDER.iterdir() if p.is_dir()] for folder in subfolders: if (folder == FolderNames.additional_info.value) or (folder == FolderNames.projection_data.value): @@ -225,6 +232,7 @@ def load_timba_data(self): @staticmethod def save_data(self): + OUTPUT_FOLDER = self.paths[PathNames.OUTPUT_FOLDER.value] for sc in self.sc_list: carbon_data_ext = DataManager.flattening_data(data=self.carbon_data[sc]) carbon_data_ext = DataManager.add_additional_info(self, data=carbon_data_ext, sc=sc) @@ -252,6 +260,7 @@ def merge_sc_data(self): @staticmethod def load_additional_data(self): + ADD_INFO_COUNTRY = self.paths[PathNames.ADD_INFO_COUNTRY.value] self.add_data["country_data"] = DataManager.load_data( f"{ADD_INFO_COUNTRY}.csv", ADD_INFO_COUNTRY, "csv") @@ -284,6 +293,8 @@ def load_additional_data_carbon(self): Additional information for projections of carbon removals and emissions are readin :param self: object of class C-Module """ + ADD_INFO_CARBON_PATH = self.paths[PathNames.ADD_INFO_CARBON_PATH.value] + commodity_code = VarNames.commodity_code.value for sheet_name in pd.ExcelFile(f"{ADD_INFO_CARBON_PATH}.xlsx").sheet_names: if "CarbonHWP_" in sheet_name: @@ -308,6 +319,8 @@ def load_faostat_data(self, update_data: bool): :param self: object of class C-Module :param update_data: Flag to update FAOSTAT data even if max cache age is not reached """ + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] + FAO_DIR = self.paths[PathNames.FAO_DIR.value] CSV_FILE = Path(f"{FAOSTAT_DATA}.csv") CACHE_MAX_AGE = 2 * 30 * 24 * 60 * 60 # 2 months @@ -338,6 +351,11 @@ def download_fao_api_data(self, database: str): :param database: Database name :return: FAOSTAT data as DataFrame """ + FRA_URL = self.paths[PathNames.FRA_URL.value] + FAOSTAT_URL = self.paths[PathNames.FAOSTAT_URL.value] + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] + FRA_DATA = self.paths[PathNames.FRA_DATA.value] + self.logger.info(f"C-Module - Download {database} data from API") if database == "FRA": database_url = FRA_URL @@ -509,6 +527,8 @@ def load_fra_data(self, update_data: bool): :param update_data: Flag to update FAOSTAT data even if max cache age is not reached """ # Paths + FRA_DATA = self.paths[PathNames.FRA_DATA.value] + FAO_DIR = self.paths[PathNames.FAO_DIR.value] CSV_FILE = Path(f"{FRA_DATA}.csv") CACHE_MAX_AGE = 2 * 30 * 24 * 60 * 60 # 2 months diff --git a/c_module/data_management/process_manager.py b/c_module/data_management/process_manager.py index e1117da..d3129d6 100644 --- a/c_module/data_management/process_manager.py +++ b/c_module/data_management/process_manager.py @@ -1,6 +1,5 @@ from c_module.data_management.data_manager import DataManager -from c_module.parameters.paths import (FAOSTAT_DATA, FRA_DATA) -from c_module.parameters.defines import (VarNames, ParamNames) +from c_module.parameters.defines import (VarNames, ParamNames, PathNames) from pathlib import Path from c_module.logic.visualisation import Carbon_DashboardPlotter @@ -38,6 +37,7 @@ def readin_carbon_process(self): @staticmethod def readin_faostat_process(self): self.logger.info("C-Module - Reading in FAOSTAT data") + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] DataManager.load_faostat_data(self, update_data=self.UserInput[ParamNames.fao_data_update.value]) if not Path(f"{FAOSTAT_DATA}_processed.pkl").is_file(): DataManager.prep_faostat_data(self) @@ -49,6 +49,7 @@ def readin_faostat_process(self): @staticmethod def readin_fra_process(self): self.logger.info("C-Module - Reading in FRA data") + FRA_DATA = self.paths[PathNames.FRA_DATA.value] # TODO implement fra processing steps DataManager.load_fra_data(self, update_data=self.UserInput[ParamNames.fao_data_update.value]) if not Path(f"{FRA_DATA}_processed.pkl").is_file(): diff --git a/c_module/logic/base_logger.py b/c_module/logic/base_logger.py index 1e044f6..0d1817b 100644 --- a/c_module/logic/base_logger.py +++ b/c_module/logic/base_logger.py @@ -3,10 +3,9 @@ from pathlib import Path import datetime as dt import logging -from c_module.parameters.paths import LOGGING_OUTPUT_FOLDER -def get_logger(user_path: Union[str, Path, None], add_on_activated: bool): +def get_logger(user_path: Union[str, Path, None], add_on_activated: bool, logging_folder): current_dt = dt.datetime.now().strftime("%Y%m%d") if add_on_activated: filename = f"{current_dt}_TiMBA.log" @@ -14,7 +13,7 @@ def get_logger(user_path: Union[str, Path, None], add_on_activated: bool): filename = rf"{current_dt}_C_Module.log" if user_path is None: - filepath = os.path.join(LOGGING_OUTPUT_FOLDER, filename) + filepath = os.path.join(logging_folder, filename) else: filepath = os.path.join(user_path, "output", filename) if not os.path.exists(filepath): diff --git a/c_module/logic/carbon_calc.py b/c_module/logic/carbon_calc.py index 7e0626e..3360fbe 100644 --- a/c_module/logic/carbon_calc.py +++ b/c_module/logic/carbon_calc.py @@ -1,5 +1,4 @@ -from c_module.parameters.defines import (VarNames, ParamNames, CarbonConstants) -from c_module.parameters.paths import (PKL_ADD_INFO_START_YEAR) +from c_module.parameters.defines import (VarNames, ParamNames, CarbonConstants, PathNames) from c_module.data_management.data_manager import DataManager import pandas as pd @@ -164,6 +163,7 @@ def calc_carbon_hwp(self): for period in timba_data[period_var].unique(): if period == 0: carbon_data = CarbonCalculator.calc_historic_carbon_hwp( + self=self, timba_data=timba_data, faostat_data=faostat_data, add_carbon_data=add_carbon_data, @@ -198,7 +198,7 @@ def calc_carbon_hwp(self): self.carbon_data[sc][VarNames.carbon_hwp.value] = carbon_data @staticmethod - def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFrame, add_data: pd.DataFrame, + def calc_historic_carbon_hwp(self, timba_data: pd.DataFrame, faostat_data: pd.DataFrame, add_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): """ Calculates the historical carbon stock in semi-finished HWP based on the production or stock-change approach as @@ -212,6 +212,7 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram :param user_input: Input from user :return: Historical carbon stocks in semi-finished HWP for all countries represented in TiMBA """ + PKL_ADD_INFO_START_YEAR = self.paths[PathNames.PKL_ADD_INFO_START_YEAR.value] carbon_factor = VarNames.carbon_factor.value half_life = VarNames.half_life.value faostat_country_code = VarNames.fao_country_code.value @@ -264,7 +265,8 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram country_spec_start_year = DataManager.restore_from_pickle(f"{PKL_ADD_INFO_START_YEAR}.pkl") else: country_spec_start_year = (CarbonCalculator.determine_start_year - (user_input=user_input, + (self=self, + user_input=user_input, faostat_data=faostat_data, add_carbon_data=add_carbon_data)) @@ -272,7 +274,8 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram if ((len(country_spec_start_year[start_year].unique()) > 1) & (user_input[ParamNames.hist_hwp_start_year.value] != "country-specific")): country_spec_start_year = (CarbonCalculator.determine_start_year - (user_input=user_input, + (self=self, + user_input=user_input, faostat_data=faostat_data, add_carbon_data=add_carbon_data)) else: @@ -700,7 +703,7 @@ def calc_domestic_feedstock(data: pd.DataFrame): return share_domestic_feedstock @staticmethod - def determine_start_year(faostat_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): + def determine_start_year(self, faostat_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): """ Determines dynamically the start year for the historic HWP carbon pool calculation based on the data availability of FAOSTAT. The start year is determined for each country and product. The start year is determined @@ -711,6 +714,7 @@ def determine_start_year(faostat_data: pd.DataFrame, add_carbon_data: pd.DataFra :param user_input: Input from user :return: DataFrame with start years """ + PKL_ADD_INFO_START_YEAR = self.paths[PathNames.PKL_ADD_INFO_START_YEAR.value] faostat_country_code = VarNames.fao_country_code.value faostat_commodity_code = VarNames.faostat_item_code.value year_var = VarNames.year_name.value diff --git a/c_module/logic/main.py b/c_module/logic/main.py index ca33287..0056388 100644 --- a/c_module/logic/main.py +++ b/c_module/logic/main.py @@ -2,7 +2,8 @@ from c_module.data_management.process_manager import ProcessManager from c_module.logic.carbon_calc import CarbonCalculator from c_module.logic.base_logger import get_logger -from c_module.parameters.defines import ParamNames +from c_module.parameters.defines import ParamNames, PathNames +from c_module.parameters.paths import set_paths class C_Module(object): @@ -10,7 +11,9 @@ def __init__(self, UserInput): self.UserInput = UserInput self.add_on_activated = UserInput[ParamNames.add_on_activated.value] self.time_stamp = dt.datetime.now().strftime("%Y%m%dT%H-%M-%S") - self.logger = get_logger(None, add_on_activated=self.add_on_activated) + self.paths = set_paths(user_input=self.UserInput) + self.logger = get_logger(None, add_on_activated=self.add_on_activated, + logging_folder=self.paths[PathNames.LOGGING_OUTPUT_FOLDER.value]) self.sc_path = [] self.sc_list = [] self.timba_data = {} diff --git a/c_module/parameters/defines.py b/c_module/parameters/defines.py index acddc29..db6d365 100644 --- a/c_module/parameters/defines.py +++ b/c_module/parameters/defines.py @@ -145,6 +145,28 @@ class FolderNames(Enum): historical_data = "historical_data" +class PathNames(Enum): + INPUT_FOLDER = "INPUT_FOLDER" + OUTPUT_FOLDER = "OUTPUT_FOLDER" + TIMBADIR_INPUT = "TIMBADIR_INPUT" + TIMBADIR_OUTPUT = "TIMBADIR_OUTPUT" + FAO_DIR = "FAO_DIR" + FAOSTAT_URL = "FAOSTAT_URL" + FAOSTAT_DATA = "FAOSTAT_DATA" + FRA_URL = "FRA_URL" + FRA_DATA = "FRA_DATA" + CMODULE_ZIP_URL = "CMODULE_ZIP_URL" + ADD_INFO_DIR = "ADD_INFO_DIR" + ADD_INFO_FOLDER = "ADD_INFO_FOLDER" + ADD_INFO_CARBON_PATH = "ADD_INFO_CARBON_PATH" + PKL_ADD_INFO_CARBON_PATH = "PKL_ADD_INFO_CARBON_PATH" + ADD_INFO_COUNTRY = "ADD_INFO_COUNTRY" + PKL_ADD_INFO_START_YEAR = "PKL_ADD_INFO_START_YEAR" + DEFAULT_PROJECTION_DIR = "DEFAULT_PROJECTION_DIR" + LOGGING_OUTPUT_FOLDER = "LOGGING_OUTPUT_FOLDER" + + + diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index a4d3187..d2f5527 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -1,7 +1,7 @@ import datetime as dt from pathlib import Path from c_module.user_io.default_parameters import user_input -from c_module.parameters.defines import ParamNames +from c_module.parameters.defines import ParamNames, PathNames current_dt = dt.datetime.now().strftime("%Y%m%dT%H-%M-%S") @@ -115,71 +115,87 @@ def cmodule_is_standalone(debug: bool = False) -> bool: return False -PACKAGEDIR = Path(__file__).parent.parent.absolute() - -if cmodule_is_standalone(debug=False): - if user_input[ParamNames.add_on_activated.value]: - import sys - print("Inconsistent settings:") - print(f"C-Module is executed as standalone: {cmodule_is_standalone(debug=False)}") - print(f"But parameter add_on_activated: {user_input[ParamNames.add_on_activated.value]}") - print(f"Harmonize settings to proceed") - sys.exit("Stopping execution.") - -if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=True): - # input and output paths for add-on c-module - if user_input[ParamNames.folderpath.value] is None: - # If user-defined path does not exists, use default path - # For compatibility with other modules, paths must be adapted - TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() - TARGETDIR = TIMBADIR - else: - # If user-defined path exist - USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() - TARGETDIR = USER_PATH - - TIMBADIR_INPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") - TIMBADIR_OUTPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") - - INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") - OUTPUT_FOLDER = TIMBADIR_OUTPUT +def set_paths(user_input: dict) -> dict: + PACKAGEDIR = Path(__file__).parent.parent.absolute() + + if cmodule_is_standalone(debug=False): + if user_input[ParamNames.add_on_activated.value]: + import sys + print("Inconsistent settings:") + print(f"C-Module is executed as standalone: {cmodule_is_standalone(debug=False)}") + print(f"But parameter add_on_activated: {user_input[ParamNames.add_on_activated.value]}") + print(f"Harmonize settings to proceed") + sys.exit("Stopping execution.") + + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): + # input and output paths for add-on c-module + if user_input[ParamNames.folderpath.value] is None: + # If user-defined path does not exists, use default path + # For compatibility with other modules, paths must be adapted + TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() + TARGETDIR = TIMBADIR + else: + # If user-defined path exist + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TARGETDIR = USER_PATH + + TIMBADIR_INPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + + INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") + OUTPUT_FOLDER = TIMBADIR_OUTPUT -else: - # input and output paths for standalone c-module - if user_input[ParamNames.folderpath.value] is None: - TARGETDIR = PACKAGEDIR else: - USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() - TARGETDIR = USER_PATH - print(f"USER PATH used: {USER_PATH}") - TIMBADIR_INPUT = None - TIMBADIR_OUTPUT = None - - INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") - OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") - -print("Input path used") -print(INPUT_FOLDER) -print("\n") -print("Output path used") -print(OUTPUT_FOLDER) - - -# Official statistics from the Food and Agriculture Organization -FAO_DIR = INPUT_FOLDER / Path("historical_data") -FAOSTAT_URL = "https://bulks-faostat.fao.org/production/Forestry_E_All_Data.zip" -FAOSTAT_DATA = INPUT_FOLDER / Path("historical_data") / Path("Forestry_E_All_Data_NOFLAG") -FRA_URL = "https://fra-data.fao.org/api/file/bulk-download?assessmentName=fra&cycleName=2020&countryIso=WO" -FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") - -# additional information -CMODULE_ZIP_URL = "https://github.com/TI-Forest-Sector-Modelling/C-Module/archive/refs/heads/main.zip" -ADD_INFO_DIR = "C-Module-main/c_module/data/input/additional_information" -ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") -ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") -PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") -ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") -PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") -DEFAULT_PROJECTION_DIR = "C-Module-main/c_module/data/input/projection_data" - -LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER + # input and output paths for standalone c-module + if user_input[ParamNames.folderpath.value] is None: + TARGETDIR = PACKAGEDIR + else: + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TARGETDIR = USER_PATH + + TIMBADIR_INPUT = None + TIMBADIR_OUTPUT = None + + INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") + OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") + + # Official statistics from the Food and Agriculture Organization + FAO_DIR = INPUT_FOLDER / Path("historical_data") + FAOSTAT_URL = "https://bulks-faostat.fao.org/production/Forestry_E_All_Data.zip" + FAOSTAT_DATA = INPUT_FOLDER / Path("historical_data") / Path("Forestry_E_All_Data_NOFLAG") + FRA_URL = "https://fra-data.fao.org/api/file/bulk-download?assessmentName=fra&cycleName=2020&countryIso=WO" + FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") + + # additional information + CMODULE_ZIP_URL = "https://github.com/TI-Forest-Sector-Modelling/C-Module/archive/refs/heads/main.zip" + ADD_INFO_DIR = "C-Module-main/c_module/data/input/additional_information" + ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") + ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") + PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") + ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") + PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") + DEFAULT_PROJECTION_DIR = "C-Module-main/c_module/data/input/projection_data" + + LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER + + path_dict = { + PathNames.INPUT_FOLDER.value: INPUT_FOLDER, + PathNames.OUTPUT_FOLDER.value: OUTPUT_FOLDER, + PathNames.TIMBADIR_INPUT.value: TIMBADIR_INPUT, + PathNames.TIMBADIR_OUTPUT.value: TIMBADIR_OUTPUT, + PathNames.FAO_DIR.value: FAO_DIR, + PathNames.FAOSTAT_URL.value: FAOSTAT_URL, + PathNames.FAOSTAT_DATA.value: FAOSTAT_DATA, + PathNames.FRA_URL.value: FRA_URL, + PathNames.FRA_DATA.value: FRA_DATA, + PathNames.CMODULE_ZIP_URL.value: CMODULE_ZIP_URL, + PathNames.ADD_INFO_DIR.value: ADD_INFO_DIR, + PathNames.ADD_INFO_FOLDER.value: ADD_INFO_FOLDER, + PathNames.ADD_INFO_CARBON_PATH.value: ADD_INFO_CARBON_PATH, + PathNames.PKL_ADD_INFO_CARBON_PATH.value: PKL_ADD_INFO_CARBON_PATH, + PathNames.ADD_INFO_COUNTRY.value: ADD_INFO_COUNTRY, + PathNames.PKL_ADD_INFO_START_YEAR.value: PKL_ADD_INFO_START_YEAR, + PathNames.DEFAULT_PROJECTION_DIR.value: DEFAULT_PROJECTION_DIR, + PathNames.LOGGING_OUTPUT_FOLDER.value: LOGGING_OUTPUT_FOLDER + } + return path_dict \ No newline at end of file diff --git a/c_module/user_io/default_parameters.py b/c_module/user_io/default_parameters.py index 711ee3e..5a3c309 100644 --- a/c_module/user_io/default_parameters.py +++ b/c_module/user_io/default_parameters.py @@ -9,7 +9,7 @@ end_year = 2050 # Not activated read_in_pkl = True # Caution False option is not implemented yet -folderpath = "C:/Users/honkomp/test_test_test" +folderpath = None # Forest carbon related parameters calc_c_forest_agb = True From b8d45fa3c4a6a81945bc0e857674c722e98ee750 Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 17:35:46 +0100 Subject: [PATCH 22/23] Activate debug trigger --- c_module/parameters/paths.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index d2f5527..34e78a6 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -127,7 +127,7 @@ def set_paths(user_input: dict) -> dict: print(f"Harmonize settings to proceed") sys.exit("Stopping execution.") - if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=True): # input and output paths for add-on c-module if user_input[ParamNames.folderpath.value] is None: # If user-defined path does not exists, use default path @@ -159,6 +159,12 @@ def set_paths(user_input: dict) -> dict: INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") + print("\n") + print(f"Input path used: {INPUT_FOLDER}") + print("\n") + print(f"Output path used: {OUTPUT_FOLDER}") + print("\n") + # Official statistics from the Food and Agriculture Organization FAO_DIR = INPUT_FOLDER / Path("historical_data") FAOSTAT_URL = "https://bulks-faostat.fao.org/production/Forestry_E_All_Data.zip" From 96403e0376bdd9cbb9b8ce738f2b756986138ccd Mon Sep 17 00:00:00 2001 From: tomke honkomp Date: Wed, 3 Dec 2025 17:47:59 +0100 Subject: [PATCH 23/23] Changed return for cmodule_is_standalone() --- c_module/parameters/paths.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 34e78a6..67afe2f 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -50,12 +50,12 @@ def cmodule_is_standalone(debug: bool = False) -> bool: reasons.append("pytest detected in sys.modules") if debug: print("DEBUG: pytest present -> treated as imported") - return False + return True if any("unittest" in mod for mod in sys.modules): reasons.append("unittest detected in sys.modules") if debug: print("DEBUG: unittest present -> treated as imported") - return False + return True # Simple and reliable check for most cases if __name__ == "__main__":