diff --git a/c_module/cli/cli.py b/c_module/cli/cli.py index cbab990..519dad9 100644 --- a/c_module/cli/cli.py +++ b/c_module/cli/cli.py @@ -6,7 +6,7 @@ @click.command() @click.option('-ADD_ON', '--add_on_activated', "add_on_activated", - default=user_input[ParamNames.add_on_activated.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.add_on_activated.value], show_default=True, required=True, is_flag=True, help="Flag to use the carbon module as a standalone module or as a TiMBA add-on.") @click.option('-SC', '--sc_num', "sc_num", default=user_input[ParamNames.sc_num.value], show_default=True, required=True, type=int, @@ -18,31 +18,38 @@ show_default=True, required=True, type=int, help="End year of carbon calculations.") @click.option('-CF_AGB', '--calc_c_forest_agb', "calc_c_forest_agb", - default=user_input[ParamNames.calc_c_forest_agb.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_agb.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for aboveground forest biomass.") @click.option('-CF_BGB', '--calc_c_forest_bgb', "calc_c_forest_bgb", - default=user_input[ParamNames.calc_c_forest_bgb.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_bgb.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for belowground forest biomass.") @click.option('-CF_S', '--calc_c_forest_soil', "calc_c_forest_soil", - default=user_input[ParamNames.calc_c_forest_soil.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_soil.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for forest soil.") @click.option('-CF_DWL', '--calc_c_forest_dwl', "calc_c_forest_dwl", - default=user_input[ParamNames.calc_c_forest_dwl.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_forest_dwl.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for dead wood and litter.") @click.option('-C_HWP', '--calc_c_hwp', "calc_c_hwp", - default=user_input[ParamNames.calc_c_hwp.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.calc_c_hwp.value], show_default=True, required=True, is_flag=True, help="Flag to activate carbon calculation for harvested wood products.") @click.option('-C_HWP_A', '--c_hwp_accounting_approach', "c_hwp_accounting_approach", default=user_input[ParamNames.c_hwp_accounting_approach.value], show_default=True, required=True, type=str, help="Flag to select the accounting approach for carbon in harvested wood products.") @click.option('-R', '--read_in_pkl', "read_in_pkl", - default=user_input[ParamNames.read_in_pkl.value], show_default=True, required=True, type=bool, + default=user_input[ParamNames.read_in_pkl.value], show_default=True, required=True, is_flag=True, help="Flag to control if pkl- or csv-files are read; reads in if True.") @click.option('-SD', '--show_carbon_dashboard', 'show_carbon_dashboard', - default=user_input[ParamNames.show_carbon_dashboard.value], show_default=True, required=False, type=bool, - help="Flag to launch carbon dashboard.") + default=user_input[ParamNames.show_carbon_dashboard.value], show_default=True, required=False, + is_flag=True, help="Flag to launch carbon dashboard.") +@click.option('-UD', '--fao_data_update', 'fao_data_update', + default=user_input[ParamNames.fao_data_update.value], show_default=True, required=False, is_flag=True, + help="Flag to update FAO data.") +@click.option('-FP', '--folderpath', 'folderpath', default=user_input[ParamNames.folderpath.value], + show_default=True, required=False, type=str, help="Path to directory with Input/Output folder.") + def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_c_forest_bgb, calc_c_forest_soil, - calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard): + calc_c_forest_dwl, calc_c_hwp, c_hwp_accounting_approach, read_in_pkl, show_carbon_dashboard, fao_data_update, + folderpath): user_input_cli = { ParamNames.add_on_activated.value: add_on_activated, @@ -57,6 +64,8 @@ def cli(add_on_activated, sc_num, start_year, end_year, calc_c_forest_agb, calc_ ParamNames.calc_c_hwp.value: calc_c_hwp, ParamNames.c_hwp_accounting_approach.value: c_hwp_accounting_approach, ParamNames.show_carbon_dashboard.value: show_carbon_dashboard, + ParamNames.fao_data_update.value: fao_data_update, + ParamNames.folderpath.value: folderpath, # Adavanced settings not available via CLI ParamNames.historical_c_hwp.value: user_input[ParamNames.historical_c_hwp.value], ParamNames.hist_hwp_start_year.value: user_input[ParamNames.hist_hwp_start_year.value], diff --git a/c_module/data_management/data_manager.py b/c_module/data_management/data_manager.py index 11a1ab5..636a6a6 100644 --- a/c_module/data_management/data_manager.py +++ b/c_module/data_management/data_manager.py @@ -1,13 +1,11 @@ -from c_module.parameters.paths import (INPUT_FOLDER, TIMBADIR_INPUT, ADD_INFO_CARBON_PATH, ADD_INFO_COUNTRY, - FAOSTAT_DATA, FRA_DATA, OUTPUT_FOLDER, TIMBADIR_OUTPUT, FAOSTAT_URL, FAO_DIR, - FRA_URL) from c_module.parameters.paths import cmodule_is_standalone, extract_scenarios -from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants) +from c_module.parameters.defines import (VarNames, ParamNames, CountryConstants, FolderNames, PathNames) from c_module.user_io.default_parameters import user_input import pandas as pd from tqdm import tqdm from pathlib import Path import requests +from io import BytesIO import zipfile import io import time @@ -18,9 +16,12 @@ class DataManager: @staticmethod def set_sc_paths(self): - if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): - # input paths for add-on c-module + TIMBADIR_INPUT = self.paths[PathNames.TIMBADIR_INPUT.value] + TIMBADIR_OUTPUT = self.paths[PathNames.TIMBADIR_OUTPUT.value] + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=False): + # input paths for add-on c-module scenarios = extract_scenarios(input_folder=TIMBADIR_INPUT, output_folder=TIMBADIR_OUTPUT, sc_num=user_input[ParamNames.sc_num.value]) @@ -35,6 +36,136 @@ def set_sc_paths(self): self.sc_path = PKL_RESULTS_INPUT + @staticmethod + def check_input_data(self): + """ + Checks input data for the C-Module in two steps. First, the input data structure is checked. After, the content + of each input data folder is checked. + """ + self.logger.info(f"C-Module - Check input data for carbon module") + DataManager.check_input_data_structure(self) + DataManager.check_input_data_content(self) + + @staticmethod + def check_input_data_structure(self): + """ + Checks the input data structure. If input data folder are missing, the missing folder is generated. + """ + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] + INPUT_FOLDER.mkdir(parents=True, exist_ok=True) + + if cmodule_is_standalone(debug=False): + required = {FolderNames.additional_info.value, FolderNames.projection_data.value} + else: + required = {FolderNames.additional_info.value} + existing = {p.name for p in Path(INPUT_FOLDER).iterdir() if p.is_dir()} + missing = list(required - existing) + if len(missing) > 0: + for missing_folder in missing: + NEW_FOLDER = INPUT_FOLDER / Path(missing_folder) + NEW_FOLDER.mkdir(parents=True, exist_ok=True) + + @staticmethod + def check_input_data_content(self): + """ + Checks if input data folder content corresponds to folder content from the C-Module main branch on GitHub. + Missing input data is downloaded automatically. + :param self: C-Module object + """ + INPUT_FOLDER = self.paths[PathNames.INPUT_FOLDER.value] + CMODULE_ZIP_URL = self.paths[PathNames.CMODULE_ZIP_URL.value] + ADD_INFO_DIR = self.paths[PathNames.ADD_INFO_DIR.value] + DEFAULT_PROJECTION_DIR = self.paths[PathNames.DEFAULT_PROJECTION_DIR.value] + + subfolders = [p.name for p in INPUT_FOLDER.iterdir() if p.is_dir()] + for folder in subfolders: + if (folder == FolderNames.additional_info.value) or (folder == FolderNames.projection_data.value): + if folder == FolderNames.additional_info.value: + # download additional info data + GIT_DATA_DIR = ADD_INFO_DIR + + if folder == FolderNames.projection_data.value: + # download projection data + GIT_DATA_DIR = DEFAULT_PROJECTION_DIR + + folder_path = INPUT_FOLDER / Path(folder) + missing_files = DataManager.compare_local_and_remote(local_folder_path=folder_path, + repo_zip_url=CMODULE_ZIP_URL, + target_subdir=GIT_DATA_DIR) + + for missing_file in list(missing_files): + DataManager.download_carbon_data_from_github(self=self, + repo_zip_url=CMODULE_ZIP_URL, + target_subdir=GIT_DATA_DIR, + folder_path=folder_path, + missing_file=missing_file) + + @staticmethod + def compare_local_and_remote(local_folder_path: Path, repo_zip_url: str, target_subdir: str): + """ + Compares local and remote input data folder and returns missing files. + :param local_folder_path: Local input data folder + :param repo_zip_url: Remote input data zip url + :param target_subdir: Target subdirectory of remote input data folder + :return: Missing files in local folder + """ + response = requests.get(repo_zip_url, timeout=60) + response.raise_for_status() + + with zipfile.ZipFile(BytesIO(response.content)) as zip_file: + zip_files = zip_file.namelist() + + # GitHub files + github_filenames = { + Path(f).name + for f in zip_files + if f.startswith(target_subdir) and not f.endswith("/") + } + + # Local files + local_filenames = { + p.name for p in local_folder_path.iterdir() if p.is_file() + } + + missing_local = github_filenames - local_filenames + + return missing_local + + @staticmethod + def download_carbon_data_from_github(self, repo_zip_url: str, target_subdir: str, folder_path: Path, + missing_file: str): + """ + Downloads missing input data from GitHub. + :param self: C-Module object + :param repo_zip_url: Remote input data zip url + :param target_subdir: Target subdirectory of remote input data folder + :param folder_path: Local input data folder + :param missing_file: Input data files missing in local folder + """ + response = requests.get(repo_zip_url, timeout=60) + response.raise_for_status() + + with zipfile.ZipFile(BytesIO(response.content)) as zip_file: + zip_files = zip_file.namelist() + + target_path = None + for f in zip_files: + if f.startswith(target_subdir) and f.endswith(missing_file): + target_path = f + break + + if not target_path: + raise FileNotFoundError( + f"{missing_file} not found in GitHub folder {target_subdir}" + ) + + self.logger.info(f"C-Module - Download {missing_file} from GitHub") + + with zip_file.open(target_path) as zf: + out_file = folder_path / missing_file + with open(out_file, "wb") as f: + f.write(zf.read()) + @staticmethod def load_data(filepath, table_name, input_source): if input_source.lower() == "excel": @@ -101,6 +232,7 @@ def load_timba_data(self): @staticmethod def save_data(self): + OUTPUT_FOLDER = self.paths[PathNames.OUTPUT_FOLDER.value] for sc in self.sc_list: carbon_data_ext = DataManager.flattening_data(data=self.carbon_data[sc]) carbon_data_ext = DataManager.add_additional_info(self, data=carbon_data_ext, sc=sc) @@ -109,12 +241,11 @@ def save_data(self): if not self.UserInput[ParamNames.add_on_activated.value]: DataManager.serialize_to_pickle(self.timba_data[sc], OUTPUT_FOLDER / Path(f"{sc}.pkl")) else: - DataManager.serialize_to_pickle( - self.carbon_data[sc], OUTPUT_FOLDER / Path(f"{self.time_stamp}_{sc}.pkl")) + DataManager.serialize_to_pickle(self.carbon_data[sc], OUTPUT_FOLDER / Path(f"{sc}.pkl")) for df_key in self.carbon_data[sc].keys(): carbon_data = self.carbon_data[sc][df_key] - carbon_data_path = OUTPUT_FOLDER / Path(f"{df_key}_D{self.time_stamp}_{sc}") + carbon_data_path = OUTPUT_FOLDER / Path(f"{df_key}_{sc}") carbon_data.to_csv(f"{carbon_data_path}.csv", index=False) @staticmethod @@ -129,6 +260,7 @@ def merge_sc_data(self): @staticmethod def load_additional_data(self): + ADD_INFO_COUNTRY = self.paths[PathNames.ADD_INFO_COUNTRY.value] self.add_data["country_data"] = DataManager.load_data( f"{ADD_INFO_COUNTRY}.csv", ADD_INFO_COUNTRY, "csv") @@ -139,7 +271,6 @@ def retrieve_commodity_num(self): commodity_dict = VarNames.commodity_dict.value commodity_code = VarNames.commodity_code.value commodity_num_name = VarNames.commodity_num.value - commodity_num = len(self.timba_data[self.sc_list[0]][timba_data_all][commodity_code].unique()) self.add_data[commodity_dict] = {} self.add_data[commodity_dict][commodity_num_name] = commodity_num @@ -162,6 +293,8 @@ def load_additional_data_carbon(self): Additional information for projections of carbon removals and emissions are readin :param self: object of class C-Module """ + ADD_INFO_CARBON_PATH = self.paths[PathNames.ADD_INFO_CARBON_PATH.value] + commodity_code = VarNames.commodity_code.value for sheet_name in pd.ExcelFile(f"{ADD_INFO_CARBON_PATH}.xlsx").sheet_names: if "CarbonHWP_" in sheet_name: @@ -186,6 +319,8 @@ def load_faostat_data(self, update_data: bool): :param self: object of class C-Module :param update_data: Flag to update FAOSTAT data even if max cache age is not reached """ + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] + FAO_DIR = self.paths[PathNames.FAO_DIR.value] CSV_FILE = Path(f"{FAOSTAT_DATA}.csv") CACHE_MAX_AGE = 2 * 30 * 24 * 60 * 60 # 2 months @@ -216,6 +351,11 @@ def download_fao_api_data(self, database: str): :param database: Database name :return: FAOSTAT data as DataFrame """ + FRA_URL = self.paths[PathNames.FRA_URL.value] + FAOSTAT_URL = self.paths[PathNames.FAOSTAT_URL.value] + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] + FRA_DATA = self.paths[PathNames.FRA_DATA.value] + self.logger.info(f"C-Module - Download {database} data from API") if database == "FRA": database_url = FRA_URL @@ -387,6 +527,8 @@ def load_fra_data(self, update_data: bool): :param update_data: Flag to update FAOSTAT data even if max cache age is not reached """ # Paths + FRA_DATA = self.paths[PathNames.FRA_DATA.value] + FAO_DIR = self.paths[PathNames.FAO_DIR.value] CSV_FILE = Path(f"{FRA_DATA}.csv") CACHE_MAX_AGE = 2 * 30 * 24 * 60 * 60 # 2 months diff --git a/c_module/data_management/process_manager.py b/c_module/data_management/process_manager.py index 65357e6..d3129d6 100644 --- a/c_module/data_management/process_manager.py +++ b/c_module/data_management/process_manager.py @@ -1,6 +1,5 @@ from c_module.data_management.data_manager import DataManager -from c_module.parameters.paths import (FAOSTAT_DATA, FRA_DATA) -from c_module.parameters.defines import (VarNames, ParamNames) +from c_module.parameters.defines import (VarNames, ParamNames, PathNames) from pathlib import Path from c_module.logic.visualisation import Carbon_DashboardPlotter @@ -8,6 +7,7 @@ class ProcessManager: @staticmethod def run_readin_process(self): + DataManager.check_input_data(self) DataManager.set_sc_paths(self) ProcessManager.readin_add_data_process(self) ProcessManager.readin_timba_process(self) @@ -37,6 +37,7 @@ def readin_carbon_process(self): @staticmethod def readin_faostat_process(self): self.logger.info("C-Module - Reading in FAOSTAT data") + FAOSTAT_DATA = self.paths[PathNames.FAOSTAT_DATA.value] DataManager.load_faostat_data(self, update_data=self.UserInput[ParamNames.fao_data_update.value]) if not Path(f"{FAOSTAT_DATA}_processed.pkl").is_file(): DataManager.prep_faostat_data(self) @@ -48,6 +49,7 @@ def readin_faostat_process(self): @staticmethod def readin_fra_process(self): self.logger.info("C-Module - Reading in FRA data") + FRA_DATA = self.paths[PathNames.FRA_DATA.value] # TODO implement fra processing steps DataManager.load_fra_data(self, update_data=self.UserInput[ParamNames.fao_data_update.value]) if not Path(f"{FRA_DATA}_processed.pkl").is_file(): diff --git a/c_module/logic/base_logger.py b/c_module/logic/base_logger.py index 1e044f6..0d1817b 100644 --- a/c_module/logic/base_logger.py +++ b/c_module/logic/base_logger.py @@ -3,10 +3,9 @@ from pathlib import Path import datetime as dt import logging -from c_module.parameters.paths import LOGGING_OUTPUT_FOLDER -def get_logger(user_path: Union[str, Path, None], add_on_activated: bool): +def get_logger(user_path: Union[str, Path, None], add_on_activated: bool, logging_folder): current_dt = dt.datetime.now().strftime("%Y%m%d") if add_on_activated: filename = f"{current_dt}_TiMBA.log" @@ -14,7 +13,7 @@ def get_logger(user_path: Union[str, Path, None], add_on_activated: bool): filename = rf"{current_dt}_C_Module.log" if user_path is None: - filepath = os.path.join(LOGGING_OUTPUT_FOLDER, filename) + filepath = os.path.join(logging_folder, filename) else: filepath = os.path.join(user_path, "output", filename) if not os.path.exists(filepath): diff --git a/c_module/logic/carbon_calc.py b/c_module/logic/carbon_calc.py index 7e0626e..3360fbe 100644 --- a/c_module/logic/carbon_calc.py +++ b/c_module/logic/carbon_calc.py @@ -1,5 +1,4 @@ -from c_module.parameters.defines import (VarNames, ParamNames, CarbonConstants) -from c_module.parameters.paths import (PKL_ADD_INFO_START_YEAR) +from c_module.parameters.defines import (VarNames, ParamNames, CarbonConstants, PathNames) from c_module.data_management.data_manager import DataManager import pandas as pd @@ -164,6 +163,7 @@ def calc_carbon_hwp(self): for period in timba_data[period_var].unique(): if period == 0: carbon_data = CarbonCalculator.calc_historic_carbon_hwp( + self=self, timba_data=timba_data, faostat_data=faostat_data, add_carbon_data=add_carbon_data, @@ -198,7 +198,7 @@ def calc_carbon_hwp(self): self.carbon_data[sc][VarNames.carbon_hwp.value] = carbon_data @staticmethod - def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFrame, add_data: pd.DataFrame, + def calc_historic_carbon_hwp(self, timba_data: pd.DataFrame, faostat_data: pd.DataFrame, add_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): """ Calculates the historical carbon stock in semi-finished HWP based on the production or stock-change approach as @@ -212,6 +212,7 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram :param user_input: Input from user :return: Historical carbon stocks in semi-finished HWP for all countries represented in TiMBA """ + PKL_ADD_INFO_START_YEAR = self.paths[PathNames.PKL_ADD_INFO_START_YEAR.value] carbon_factor = VarNames.carbon_factor.value half_life = VarNames.half_life.value faostat_country_code = VarNames.fao_country_code.value @@ -264,7 +265,8 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram country_spec_start_year = DataManager.restore_from_pickle(f"{PKL_ADD_INFO_START_YEAR}.pkl") else: country_spec_start_year = (CarbonCalculator.determine_start_year - (user_input=user_input, + (self=self, + user_input=user_input, faostat_data=faostat_data, add_carbon_data=add_carbon_data)) @@ -272,7 +274,8 @@ def calc_historic_carbon_hwp(timba_data: pd.DataFrame, faostat_data: pd.DataFram if ((len(country_spec_start_year[start_year].unique()) > 1) & (user_input[ParamNames.hist_hwp_start_year.value] != "country-specific")): country_spec_start_year = (CarbonCalculator.determine_start_year - (user_input=user_input, + (self=self, + user_input=user_input, faostat_data=faostat_data, add_carbon_data=add_carbon_data)) else: @@ -700,7 +703,7 @@ def calc_domestic_feedstock(data: pd.DataFrame): return share_domestic_feedstock @staticmethod - def determine_start_year(faostat_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): + def determine_start_year(self, faostat_data: pd.DataFrame, add_carbon_data: pd.DataFrame, user_input: dict): """ Determines dynamically the start year for the historic HWP carbon pool calculation based on the data availability of FAOSTAT. The start year is determined for each country and product. The start year is determined @@ -711,6 +714,7 @@ def determine_start_year(faostat_data: pd.DataFrame, add_carbon_data: pd.DataFra :param user_input: Input from user :return: DataFrame with start years """ + PKL_ADD_INFO_START_YEAR = self.paths[PathNames.PKL_ADD_INFO_START_YEAR.value] faostat_country_code = VarNames.fao_country_code.value faostat_commodity_code = VarNames.faostat_item_code.value year_var = VarNames.year_name.value diff --git a/c_module/logic/main.py b/c_module/logic/main.py index ca33287..0056388 100644 --- a/c_module/logic/main.py +++ b/c_module/logic/main.py @@ -2,7 +2,8 @@ from c_module.data_management.process_manager import ProcessManager from c_module.logic.carbon_calc import CarbonCalculator from c_module.logic.base_logger import get_logger -from c_module.parameters.defines import ParamNames +from c_module.parameters.defines import ParamNames, PathNames +from c_module.parameters.paths import set_paths class C_Module(object): @@ -10,7 +11,9 @@ def __init__(self, UserInput): self.UserInput = UserInput self.add_on_activated = UserInput[ParamNames.add_on_activated.value] self.time_stamp = dt.datetime.now().strftime("%Y%m%dT%H-%M-%S") - self.logger = get_logger(None, add_on_activated=self.add_on_activated) + self.paths = set_paths(user_input=self.UserInput) + self.logger = get_logger(None, add_on_activated=self.add_on_activated, + logging_folder=self.paths[PathNames.LOGGING_OUTPUT_FOLDER.value]) self.sc_path = [] self.sc_list = [] self.timba_data = {} diff --git a/c_module/parameters/defines.py b/c_module/parameters/defines.py index 9bbb764..db6d365 100644 --- a/c_module/parameters/defines.py +++ b/c_module/parameters/defines.py @@ -28,6 +28,7 @@ class ParamNames(Enum): start_year = "start_year" end_year = "end_year" read_in_pkl = "read_in_pkl" + folderpath = "folderpath" save_data_as = "save_data_as" calc_c_forest_agb = "calc_c_forest_agb" calc_c_forest_bgb = "calc_c_forest_bgb" @@ -137,6 +138,36 @@ class VarNames(Enum): faostat_export_value = "Export value" +class FolderNames(Enum): + # Folder names + additional_info = "additional_information" + projection_data = "projection_data" + historical_data = "historical_data" + + +class PathNames(Enum): + INPUT_FOLDER = "INPUT_FOLDER" + OUTPUT_FOLDER = "OUTPUT_FOLDER" + TIMBADIR_INPUT = "TIMBADIR_INPUT" + TIMBADIR_OUTPUT = "TIMBADIR_OUTPUT" + FAO_DIR = "FAO_DIR" + FAOSTAT_URL = "FAOSTAT_URL" + FAOSTAT_DATA = "FAOSTAT_DATA" + FRA_URL = "FRA_URL" + FRA_DATA = "FRA_DATA" + CMODULE_ZIP_URL = "CMODULE_ZIP_URL" + ADD_INFO_DIR = "ADD_INFO_DIR" + ADD_INFO_FOLDER = "ADD_INFO_FOLDER" + ADD_INFO_CARBON_PATH = "ADD_INFO_CARBON_PATH" + PKL_ADD_INFO_CARBON_PATH = "PKL_ADD_INFO_CARBON_PATH" + ADD_INFO_COUNTRY = "ADD_INFO_COUNTRY" + PKL_ADD_INFO_START_YEAR = "PKL_ADD_INFO_START_YEAR" + DEFAULT_PROJECTION_DIR = "DEFAULT_PROJECTION_DIR" + LOGGING_OUTPUT_FOLDER = "LOGGING_OUTPUT_FOLDER" + + + + diff --git a/c_module/parameters/paths.py b/c_module/parameters/paths.py index 6e9b801..67afe2f 100644 --- a/c_module/parameters/paths.py +++ b/c_module/parameters/paths.py @@ -1,7 +1,7 @@ import datetime as dt from pathlib import Path from c_module.user_io.default_parameters import user_input -from c_module.parameters.defines import ParamNames +from c_module.parameters.defines import ParamNames, PathNames current_dt = dt.datetime.now().strftime("%Y%m%dT%H-%M-%S") @@ -14,76 +14,194 @@ def extract_scenarios(input_folder, output_folder, sc_num): :param sc_num: Number of scenarios to extract. :return: List of merged scenario names. """ - scenarios = [] + folder_path = Path(output_folder) + files = list(folder_path.glob("*.pkl")) + files.sort(key=lambda f: f.stat().st_mtime) if sc_num is None: folder_path = Path(input_folder) - for file in folder_path.glob("*.xlsx"): - scenario_name = f"DataContainer_Sc_{file.stem}.pkl" - scenario_path = Path(output_folder) / Path(scenario_name) - scenarios.append(scenario_path) - + try: + sc_num = len(list(folder_path.glob("*.xlsx"))) + except FileNotFoundError: + sc_num = 1 + files = files[-sc_num:] else: - folder_path = Path(output_folder) - files = list(folder_path.glob("*.pkl")) - files.sort(key=lambda f: f.stat().st_mtime) files = files[-user_input[ParamNames.sc_num.value]:] - scenarios = files + + scenarios = files return scenarios -def cmodule_is_standalone(): +def cmodule_is_standalone(debug: bool = False) -> bool: """ Check if cmodule is standalone or not, covering if the code is run as the main program, covering CLI, script, IDE, and entry point runs. + :param debug: Flag to enable debug mode. :return: Bool if cmodule is standalone or not. """ - import __main__ import sys + import inspect + import __main__ - if getattr(__main__, "__file__", None): - main_file = Path(__main__.__file__).resolve() - package_root = Path(__file__).resolve().parents[1] - - if package_root in main_file.parents: - return True - - if "pytest" in sys.modules and Path.cwd().resolve() == package_root.parent: - return True - - if any("unittest" in mod for mod in sys.modules): + reasons = [] + + # Running under a typical test runner => treat as imported + if "pytest" in sys.modules: + reasons.append("pytest detected in sys.modules") + if debug: + print("DEBUG: pytest present -> treated as imported") + return True + if any("unittest" in mod for mod in sys.modules): + reasons.append("unittest detected in sys.modules") + if debug: + print("DEBUG: unittest present -> treated as imported") + return True + + # Simple and reliable check for most cases + if __name__ == "__main__": + reasons.append("__name__ == '__main__'") + if debug: + print("DEBUG: __name__ == '__main__' -> standalone") + return True + + # Inspect stack: some IDEs or runners execute a wrapper that sets __name__ == '__main__' + # in a different frame. If any frame was executed as __main__, assume standalone entry. + for frame_info in inspect.stack(): + g = frame_info.frame.f_globals + frame_name = g.get("__name__") + frame_file = g.get("__file__", None) + if frame_name == "__main__": + reasons.append(f"found frame with __name__ == '__main__' (file={frame_file})") + if debug: + print("DEBUG: stack frame with __name__ == '__main__' -> standalone") + print(f"DEBUG: frame file: {frame_file}") return True + # Compare the top-level script path with this package path: + # if the top-level entry script is outside this package, it likely invoked/imported the package. + main_file = getattr(__main__, "__file__", None) + if main_file: + try: + main_path = Path(main_file).resolve() + package_root = Path(__file__).resolve().parents[1] + # If the top-level script is the module file itself -> standalone + if main_path == Path(__file__).resolve(): + reasons.append("main_file equals this module file") + if debug: + print("DEBUG: main_file equals this module file -> standalone") + return True + # If the top-level script is *inside* the package: often still a standalone run (python -m) + if package_root in main_path.parents: + reasons.append("main_file is located inside package root (likely -m or IDE module run)") + if debug: + print("DEBUG: main_file inside package root -> standalone") + print(f"DEBUG: main_file={main_path}, package_root={package_root}") + return True + # Otherwise treat as imported + reasons.append("main_file exists but is outside package -> treated as imported") + if debug: + print("DEBUG: main_file outside package -> treated as imported") + print(f"DEBUG: main_file={main_path}, package_root={package_root}") + return False + except Exception as e: + # fallback + if debug: + print("DEBUG: error resolving main_file or package_root:", e) + pass + + # If none of the above matched, assume imported + if debug: + print("DEBUG: no indication of standalone execution; reasons:", reasons) return False -PACKAGEDIR = Path(__file__).parent.parent.absolute() -TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() -TIMBADIR_INPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") -TIMBADIR_OUTPUT = TIMBADIR / Path("TiMBA") / Path("data") / Path("output") -INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") - -if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(): - # output paths for add-on c-module - OUTPUT_FOLDER = TIMBADIR_OUTPUT - -else: - # output paths for standalone c-module - OUTPUT_FOLDER = PACKAGEDIR / Path("data") / Path("output") - +def set_paths(user_input: dict) -> dict: + PACKAGEDIR = Path(__file__).parent.parent.absolute() + + if cmodule_is_standalone(debug=False): + if user_input[ParamNames.add_on_activated.value]: + import sys + print("Inconsistent settings:") + print(f"C-Module is executed as standalone: {cmodule_is_standalone(debug=False)}") + print(f"But parameter add_on_activated: {user_input[ParamNames.add_on_activated.value]}") + print(f"Harmonize settings to proceed") + sys.exit("Stopping execution.") + + if user_input[ParamNames.add_on_activated.value] or not cmodule_is_standalone(debug=True): + # input and output paths for add-on c-module + if user_input[ParamNames.folderpath.value] is None: + # If user-defined path does not exists, use default path + # For compatibility with other modules, paths must be adapted + TIMBADIR = Path(__file__).parent.parent.parent.parent.parent.parent.absolute() + TARGETDIR = TIMBADIR + else: + # If user-defined path exist + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TARGETDIR = USER_PATH + + TIMBADIR_INPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("input") / Path("01_Input_Files") + TIMBADIR_OUTPUT = TARGETDIR / Path("TiMBA") / Path("data") / Path("output") / Path("data") + + INPUT_FOLDER = PACKAGEDIR / Path("data") / Path("input") + OUTPUT_FOLDER = TIMBADIR_OUTPUT -# Official statistics from the Food and Agriculture Organization -FAO_DIR = INPUT_FOLDER / Path("historical_data") -FAOSTAT_URL = "https://bulks-faostat.fao.org/production/Forestry_E_All_Data.zip" -FAOSTAT_DATA = INPUT_FOLDER / Path("historical_data") / Path("Forestry_E_All_Data_NOFLAG") -FRA_URL = "https://fra-data.fao.org/api/file/bulk-download?assessmentName=fra&cycleName=2020&countryIso=WO" -FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") - -# additional information -ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") -ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") -PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") -ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") -PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") - -LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER + else: + # input and output paths for standalone c-module + if user_input[ParamNames.folderpath.value] is None: + TARGETDIR = PACKAGEDIR + else: + USER_PATH = Path(user_input[ParamNames.folderpath.value]).absolute() + TARGETDIR = USER_PATH + + TIMBADIR_INPUT = None + TIMBADIR_OUTPUT = None + + INPUT_FOLDER = TARGETDIR / Path("data") / Path("input") + OUTPUT_FOLDER = TARGETDIR / Path("data") / Path("output") + + print("\n") + print(f"Input path used: {INPUT_FOLDER}") + print("\n") + print(f"Output path used: {OUTPUT_FOLDER}") + print("\n") + + # Official statistics from the Food and Agriculture Organization + FAO_DIR = INPUT_FOLDER / Path("historical_data") + FAOSTAT_URL = "https://bulks-faostat.fao.org/production/Forestry_E_All_Data.zip" + FAOSTAT_DATA = INPUT_FOLDER / Path("historical_data") / Path("Forestry_E_All_Data_NOFLAG") + FRA_URL = "https://fra-data.fao.org/api/file/bulk-download?assessmentName=fra&cycleName=2020&countryIso=WO" + FRA_DATA = INPUT_FOLDER / Path("historical_data") / Path(f"FRA_Years_All_Data") + + # additional information + CMODULE_ZIP_URL = "https://github.com/TI-Forest-Sector-Modelling/C-Module/archive/refs/heads/main.zip" + ADD_INFO_DIR = "C-Module-main/c_module/data/input/additional_information" + ADD_INFO_FOLDER = PACKAGEDIR / INPUT_FOLDER / Path("additional_information") + ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") + PKL_ADD_INFO_CARBON_PATH = ADD_INFO_FOLDER / Path("carbon_additional_information") + ADD_INFO_COUNTRY = ADD_INFO_FOLDER / Path("country_data") + PKL_ADD_INFO_START_YEAR = ADD_INFO_FOLDER / Path("hist_hwp_carbon_start_year") + DEFAULT_PROJECTION_DIR = "C-Module-main/c_module/data/input/projection_data" + + LOGGING_OUTPUT_FOLDER = OUTPUT_FOLDER + + path_dict = { + PathNames.INPUT_FOLDER.value: INPUT_FOLDER, + PathNames.OUTPUT_FOLDER.value: OUTPUT_FOLDER, + PathNames.TIMBADIR_INPUT.value: TIMBADIR_INPUT, + PathNames.TIMBADIR_OUTPUT.value: TIMBADIR_OUTPUT, + PathNames.FAO_DIR.value: FAO_DIR, + PathNames.FAOSTAT_URL.value: FAOSTAT_URL, + PathNames.FAOSTAT_DATA.value: FAOSTAT_DATA, + PathNames.FRA_URL.value: FRA_URL, + PathNames.FRA_DATA.value: FRA_DATA, + PathNames.CMODULE_ZIP_URL.value: CMODULE_ZIP_URL, + PathNames.ADD_INFO_DIR.value: ADD_INFO_DIR, + PathNames.ADD_INFO_FOLDER.value: ADD_INFO_FOLDER, + PathNames.ADD_INFO_CARBON_PATH.value: ADD_INFO_CARBON_PATH, + PathNames.PKL_ADD_INFO_CARBON_PATH.value: PKL_ADD_INFO_CARBON_PATH, + PathNames.ADD_INFO_COUNTRY.value: ADD_INFO_COUNTRY, + PathNames.PKL_ADD_INFO_START_YEAR.value: PKL_ADD_INFO_START_YEAR, + PathNames.DEFAULT_PROJECTION_DIR.value: DEFAULT_PROJECTION_DIR, + PathNames.LOGGING_OUTPUT_FOLDER.value: LOGGING_OUTPUT_FOLDER + } + return path_dict \ No newline at end of file diff --git a/c_module/user_io/default_parameters.py b/c_module/user_io/default_parameters.py index 4e25614..5a3c309 100644 --- a/c_module/user_io/default_parameters.py +++ b/c_module/user_io/default_parameters.py @@ -9,6 +9,7 @@ end_year = 2050 # Not activated read_in_pkl = True # Caution False option is not implemented yet +folderpath = None # Forest carbon related parameters calc_c_forest_agb = True @@ -35,6 +36,7 @@ ParamNames.start_year.value: start_year, ParamNames.end_year.value: end_year, ParamNames.read_in_pkl.value: read_in_pkl, + ParamNames.folderpath.value: folderpath, ParamNames.calc_c_forest_agb.value: calc_c_forest_agb, ParamNames.calc_c_forest_bgb.value: calc_c_forest_bgb, ParamNames.calc_c_forest_soil.value: calc_c_forest_soil,