diff --git a/data_adapter/preprocessing.py b/data_adapter/preprocessing.py index 7cc6733..d7b2ee2 100644 --- a/data_adapter/preprocessing.py +++ b/data_adapter/preprocessing.py @@ -287,6 +287,7 @@ def __unpack_bandwidths(self, df: pd.DataFrame) -> pd.DataFrame: @staticmethod def __filter_subprocess(df: pd.DataFrame, subprocess: str) -> pd.DataFrame: df = df[df["type"] == subprocess] + df = df.dropna(axis=1, how="all") return df.drop("type", axis=1) @staticmethod diff --git a/data_adapter/structure.py b/data_adapter/structure.py index 4c6375c..85e175e 100644 --- a/data_adapter/structure.py +++ b/data_adapter/structure.py @@ -179,11 +179,42 @@ def get_nodes(nodes_raw): nodes += [node for node in nodes_raw_stripped.split(",") if node != ""] return nodes + def process_emission_constraint_inputs(processes_raw): + """If there's a row in the table where the process is `ind_constraint_co2eq`, + take all unique values in the `output` column that start with `emi` and + place them into the `input` field of that row. + + refer: https://github.com/sedos-project/data_adapter/pull/39 + """ + + ind_constraint_row = processes_raw[processes_raw["process"] == "ind_constraint_co2eq"] + + if ind_constraint_row.empty: + return processes_raw + + emi_outputs = [] + for output in processes_raw["output"]: + parts = output.split(",") + for part in parts: + part = part.strip() + if part.startswith("emi_"): + emi_outputs.append(part) + + emi_outputs = list(set(emi_outputs)) + emi_output_str = ",".join(emi_outputs) + + if len(emi_outputs) == 0: + return processes_raw + + processes_raw.loc[processes_raw["process"] == "ind_constraint_co2eq", "input"] = emi_output_str + return processes_raw + processes_raw = pd.read_excel( io=self.structure_file, sheet_name=process_sheet, usecols=("process", "input", "output"), ) + wb = load_workbook(self.structure_file, read_only=True) if helper_sheet in wb.sheetnames: helpers_raw = pd.read_excel( @@ -193,8 +224,10 @@ def get_nodes(nodes_raw): ) processes_raw = pd.concat([processes_raw, helpers_raw]) processes_raw = processes_raw.fillna("") + processes_raw = process_emission_constraint_inputs(processes_raw) check_character_convention(processes_raw, ["process"]) processes = processes_raw.to_dict(orient="records") + return { process["process"]: {"inputs": get_nodes(process["input"]), "outputs": get_nodes(process["output"])} for process in processes diff --git a/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_new_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_new_1.json b/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_new_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_new_1.json index db428d8..afcc13b 100644 --- a/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_new_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_new_1.json +++ b/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_new_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_new_1.json @@ -1 +1,263 @@ -{"id": null, "@id": null, "name": "x2x_delivery_hydrogen_pipeline_new_1", "title": "sedos_x2x_delivery_hydrogen_pipeline_new_1", "review": null, "context": {"contact": "g.mueller@fz-juelich.de", "grantNo": "03EI1040D", "homepage": "https://sedos-project.github.io/.github/", "sourceCode": "https://github.com/sedos-project", "documentation": "https://sedos-project.github.io/.github/", "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)", "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg"}, "sources": [{"path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765", "title": "Determining the lowest-cost hydrogen delivery mode", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.", "instruction": null}], "description": null}, {"path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf", "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright. Frontier Economics, 2021.", "instruction": null}], "description": null}], "spatial": {"extent": null, "location": "global", "resolution": null}, "subject": [{"name": "hydrogen", "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220"}, {"name": "pipeline", "path": ""}], "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json", "_comment": {"null": "If not applicable use: null", "todo": "If a value is not yet available, use: todo", "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", "units": "Use a space between numbers and units (100 m)", "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)", "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)", "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)"}, "keywords": ["SEDOS", "AP5", "X2X", "oedatamodel-parameter", "input_data"], "language": ["en-GB"], "licenses": [{"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode", "title": "Creative Commons Zero v1.0 Universal", "attribution": null, "instruction": "You are free: To Share, To Create, To Adapt"}], "temporal": {"timeseries": [{"end": null, "start": null, "alignment": null, "resolution": null, "aggregationType": null}], "referenceDate": null}, "resources": [{"name": "model_draft.x2x_delivery_hydrogen_pipeline_new_1", "path": null, "format": "PostgreSQL", "schema": {"fields": [{"name": "id", "type": "bigint", "unit": null, "isAbout": [], "description": "Unique identifier", "valueReference": []}, {"name": "region", "type": "text", "unit": null, "isAbout": [], "description": "Country or region", "valueReference": []}, {"name": "year", "type": "integer", "unit": "a", "isAbout": [], "description": "Year", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen_orig", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen before being transported", "valueReference": []}, {"name": "conversion_factor_sec_elec", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of electricity demand for pipeline transport.", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen after being transported", "valueReference": []}, {"name": "cost_inv_p", "type": "float array", "unit": "\u20ac/MW", "isAbout": [], "description": "Investment costs per unit capacity", "valueReference": []}, {"name": "cost_fix_p", "type": "float array", "unit": "\u20ac/MW*a", "isAbout": [], "description": "Fixed operational costs per unit capacity", "valueReference": []}, {"name": "lifetime", "type": "float array", "unit": "a", "isAbout": [], "description": "Lifetime in operation.", "valueReference": []}, {"name": "wacc", "type": "text", "unit": "%", "isAbout": [], "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.", "valueReference": []}, {"name": "bandwidth_type", "type": "json", "unit": null, "isAbout": [], "description": "Bandwidth Type", "valueReference": []}, {"name": "version", "type": "text", "unit": null, "isAbout": [], "description": "Version", "valueReference": []}, {"name": "method", "type": "json", "unit": null, "isAbout": [], "description": "Method", "valueReference": []}, {"name": "source", "type": "json", "unit": null, "isAbout": [], "description": "Source", "valueReference": []}, {"name": "comment", "type": "text", "unit": null, "isAbout": [], "description": "Comment", "valueReference": []}], "primaryKey": ["id"], "foreignKeys": []}, "dialect": {"delimiter": ";", "decimalSeparator": "."}, "profile": "tabular-data-resource", "encoding": "UTF-8"}], "description": "Techno-economic parameter for newly built hydrogen pipelines in Germany.", "contributors": [{"date": "2024-04-08", "email": "g.mueller@fz-juelich.de", "title": "Gian M\u00fcller, FZJ", "object": "data and metadata", "comment": "Data for transport process of hydrogen by newly built pipeline in Germany."}], "metaMetadata": {"metadataLicense": {"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/", "title": "Creative Commons Zero v1.0 Universal"}, "metadataVersion": "OEP-1.5.2"}, "publicationDate": "2024-04-29"} \ No newline at end of file +{ + "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json", + "@id": null, + "_comment": { + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)", + "null": "If not applicable use: null", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)", + "todo": "If a value is not yet available, use: todo", + "units": "Use a space between numbers and units (100 m)" + }, + "context": { + "contact": "g.mueller@fz-juelich.de", + "documentation": "https://sedos-project.github.io/.github/", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)", + "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg", + "grantNo": "03EI1040D", + "homepage": "https://sedos-project.github.io/.github/", + "sourceCode": "https://github.com/sedos-project" + }, + "contributors": [ + { + "comment": "Data for transport process of hydrogen by newly built pipeline in Germany.", + "date": "2024-04-08", + "email": "g.mueller@fz-juelich.de", + "object": "data and metadata", + "title": "Gian M\u00fcller, FZJ" + } + ], + "description": "Techno-economic parameter for newly built hydrogen pipelines in Germany.", + "id": null, + "keywords": [ + "SEDOS", + "AP5", + "X2X", + "oedatamodel-parameter", + "input_data" + ], + "language": [ + "en-GB" + ], + "licenses": [ + { + "attribution": null, + "instruction": "You are free: To Share, To Create, To Adapt", + "name": "CC0-1.0", + "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode", + "title": "Creative Commons Zero v1.0 Universal" + } + ], + "metaMetadata": { + "metadataLicense": { + "name": "CC0-1.0", + "path": "https://creativecommons.org/publicdomain/zero/1.0/", + "title": "Creative Commons Zero v1.0 Universal" + }, + "metadataVersion": "OEP-1.5.2" + }, + "name": "x2x_delivery_hydrogen_pipeline_new_1", + "publicationDate": "2024-04-29", + "resources": [ + { + "dialect": { + "decimalSeparator": ".", + "delimiter": ";" + }, + "encoding": "UTF-8", + "format": "PostgreSQL", + "name": "model_draft.x2x_delivery_hydrogen_pipeline_new_1", + "path": null, + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "description": "Unique identifier", + "isAbout": [], + "name": "id", + "type": "bigint", + "unit": null, + "valueReference": [] + }, + { + "description": "Country or region", + "isAbout": [], + "name": "region", + "type": "text", + "unit": null, + "valueReference": [] + }, + { + "description": "Year", + "isAbout": [], + "name": "year", + "type": "integer", + "unit": "a", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of hydrogen before being transported", + "isAbout": [], + "name": "conversion_factor_sec_hydrogen_orig", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of electricity demand for pipeline transport.", + "isAbout": [], + "name": "conversion_factor_sec_elec", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of hydrogen after being transported", + "isAbout": [], + "name": "conversion_factor_sec_hydrogen", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Investment costs per unit capacity", + "isAbout": [], + "name": "cost_inv_p", + "type": "float array", + "unit": "\u20ac/MW", + "valueReference": [] + }, + { + "description": "Fixed operational costs per unit capacity", + "isAbout": [], + "name": "cost_fix_p", + "type": "float array", + "unit": "\u20ac/MW*a", + "valueReference": [] + }, + { + "description": "Lifetime in operation.", + "isAbout": [], + "name": "lifetime", + "type": "float array", + "unit": "a", + "valueReference": [] + }, + { + "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.", + "isAbout": [], + "name": "wacc", + "type": "text", + "unit": "%", + "valueReference": [] + }, + { + "description": "Bandwidth Type", + "isAbout": [], + "name": "bandwidth_type", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Version", + "isAbout": [], + "name": "version", + "type": "text", + "unit": null, + "valueReference": [] + }, + { + "description": "Method", + "isAbout": [], + "name": "method", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Source", + "isAbout": [], + "name": "source", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Comment", + "isAbout": [], + "name": "comment", + "type": "text", + "unit": null, + "valueReference": [] + } + ], + "foreignKeys": [], + "primaryKey": [ + "id" + ] + } + } + ], + "review": null, + "sources": [ + { + "description": null, + "licenses": [ + { + "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.", + "instruction": null, + "name": null, + "path": null, + "title": null + } + ], + "path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765", + "title": "Determining the lowest-cost hydrogen delivery mode" + }, + { + "description": null, + "licenses": [ + { + "attribution": "Copyright. Frontier Economics, 2021.", + "instruction": null, + "name": null, + "path": null, + "title": null + } + ], + "path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf", + "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines" + } + ], + "spatial": { + "extent": null, + "location": "global", + "resolution": null + }, + "subject": [ + { + "name": "hydrogen", + "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220" + }, + { + "name": "pipeline", + "path": "" + } + ], + "temporal": { + "referenceDate": null, + "timeseries": [ + { + "aggregationType": null, + "alignment": null, + "end": null, + "resolution": null, + "start": null + } + ] + }, + "title": "sedos_x2x_delivery_hydrogen_pipeline_new_1" +} diff --git a/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_retrofit_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_retrofit_1.json b/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_retrofit_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_retrofit_1.json index 77c1efa..4d193d0 100644 --- a/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_retrofit_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_retrofit_1.json +++ b/tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_retrofit_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_retrofit_1.json @@ -1 +1,285 @@ -{"id": null, "@id": null, "name": "x2x_delivery_hydrogen_pipeline_retrofit_1", "title": "sedos_x2x_delivery_hydrogen_pipeline_retrofit_1", "review": null, "context": {"contact": "g.mueller@fz-juelich.de", "grantNo": "03EI1040D", "homepage": "https://sedos-project.github.io/.github/", "sourceCode": "https://github.com/sedos-project", "documentation": "https://sedos-project.github.io/.github/", "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)", "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg"}, "sources": [{"path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765", "title": "Determining the lowest-cost hydrogen delivery mode", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.", "instruction": null}], "description": null}, {"path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf", "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright. Frontier Economics, 2021.", "instruction": null}], "description": null}, {"path": "https://www.dvgw.de/medien/dvgw/leistungen/publikationen/gasnetz-rueckgrat-h2-welt.pdf", "title": "Das Gasnetz - R\u00fcckgrat der Wasserstoffwelt", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright. DVGW Bonn, 2023.", "instruction": null}], "description": null}], "spatial": {"extent": null, "location": "global", "resolution": null}, "subject": [{"name": "hydrogen", "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220"}, {"name": "pipeline", "path": ""}], "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json", "_comment": {"null": "If not applicable use: null", "todo": "If a value is not yet available, use: todo", "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", "units": "Use a space between numbers and units (100 m)", "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)", "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)", "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)"}, "keywords": ["SEDOS", "AP5", "X2X", "oedatamodel-parameter", "input_data"], "language": ["en-GB"], "licenses": [{"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode", "title": "Creative Commons Zero v1.0 Universal", "attribution": null, "instruction": "You are free: To Share, To Create, To Adapt"}], "temporal": {"timeseries": [{"end": null, "start": null, "alignment": null, "resolution": null, "aggregationType": null}], "referenceDate": null}, "resources": [{"name": "model_draft.x2x_delivery_hydrogen_pipeline_retrofit_1", "path": null, "format": "PostgreSQL", "schema": {"fields": [{"name": "id", "type": "bigint", "unit": null, "isAbout": [], "description": "Unique identifier", "valueReference": []}, {"name": "region", "type": "text", "unit": null, "isAbout": [], "description": "Country or region", "valueReference": []}, {"name": "year", "type": "integer", "unit": "a", "isAbout": [], "description": "Year", "valueReference": []}, {"name": "capacity_p_max", "type": "float array", "unit": "MW", "isAbout": [], "description": "Maximum allowed throughput power output capacity per process.", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen_orig", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen before being transported", "valueReference": []}, {"name": "conversion_factor_sec_elec", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of electricity demand for pipeline transport.", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen after being transported", "valueReference": []}, {"name": "cost_inv_p", "type": "float array", "unit": "\u20ac/MW", "isAbout": [], "description": "Investment costs per unit capacity", "valueReference": []}, {"name": "cost_fix_p", "type": "float array", "unit": "\u20ac/MW*a", "isAbout": [], "description": "Fixed operational costs per unit capacity", "valueReference": []}, {"name": "lifetime", "type": "float array", "unit": "a", "isAbout": [], "description": "Lifetime in operation.", "valueReference": []}, {"name": "wacc", "type": "text", "unit": "%", "isAbout": [], "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.", "valueReference": []}, {"name": "bandwidth_type", "type": "json", "unit": null, "isAbout": [], "description": "Bandwidth Type", "valueReference": []}, {"name": "version", "type": "text", "unit": null, "isAbout": [], "description": "Version", "valueReference": []}, {"name": "method", "type": "json", "unit": null, "isAbout": [], "description": "Method", "valueReference": []}, {"name": "source", "type": "json", "unit": null, "isAbout": [], "description": "Source", "valueReference": []}, {"name": "comment", "type": "text", "unit": null, "isAbout": [], "description": "Comment", "valueReference": []}], "primaryKey": ["id"], "foreignKeys": []}, "dialect": {"delimiter": ";", "decimalSeparator": "."}, "profile": "tabular-data-resource", "encoding": "UTF-8"}], "description": "Techno-economic parameter for hydrogen pipelines that were retrofitted from natural gas pipelines in Germany.", "contributors": [{"date": "2024-04-29", "email": "g.mueller@fz-juelich.de", "title": "Gian M\u00fcller, FZJ", "object": "data and metadata", "comment": "Data for transport process of hydrogen by retrofitted pipeline in Germany."}], "metaMetadata": {"metadataLicense": {"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/", "title": "Creative Commons Zero v1.0 Universal"}, "metadataVersion": "OEP-1.5.2"}, "publicationDate": "2024-04-08"} \ No newline at end of file +{ + "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json", + "@id": null, + "_comment": { + "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", + "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)", + "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", + "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)", + "null": "If not applicable use: null", + "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)", + "todo": "If a value is not yet available, use: todo", + "units": "Use a space between numbers and units (100 m)" + }, + "context": { + "contact": "g.mueller@fz-juelich.de", + "documentation": "https://sedos-project.github.io/.github/", + "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)", + "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg", + "grantNo": "03EI1040D", + "homepage": "https://sedos-project.github.io/.github/", + "sourceCode": "https://github.com/sedos-project" + }, + "contributors": [ + { + "comment": "Data for transport process of hydrogen by retrofitted pipeline in Germany.", + "date": "2024-04-29", + "email": "g.mueller@fz-juelich.de", + "object": "data and metadata", + "title": "Gian M\u00fcller, FZJ" + } + ], + "description": "Techno-economic parameter for hydrogen pipelines that were retrofitted from natural gas pipelines in Germany.", + "id": null, + "keywords": [ + "SEDOS", + "AP5", + "X2X", + "oedatamodel-parameter", + "input_data" + ], + "language": [ + "en-GB" + ], + "licenses": [ + { + "attribution": null, + "instruction": "You are free: To Share, To Create, To Adapt", + "name": "CC0-1.0", + "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode", + "title": "Creative Commons Zero v1.0 Universal" + } + ], + "metaMetadata": { + "metadataLicense": { + "name": "CC0-1.0", + "path": "https://creativecommons.org/publicdomain/zero/1.0/", + "title": "Creative Commons Zero v1.0 Universal" + }, + "metadataVersion": "OEP-1.5.2" + }, + "name": "x2x_delivery_hydrogen_pipeline_retrofit_1", + "publicationDate": "2024-04-08", + "resources": [ + { + "dialect": { + "decimalSeparator": ".", + "delimiter": ";" + }, + "encoding": "UTF-8", + "format": "PostgreSQL", + "name": "model_draft.x2x_delivery_hydrogen_pipeline_retrofit_1", + "path": null, + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "description": "Unique identifier", + "isAbout": [], + "name": "id", + "type": "bigint", + "unit": null, + "valueReference": [] + }, + { + "description": "Country or region", + "isAbout": [], + "name": "region", + "type": "text", + "unit": null, + "valueReference": [] + }, + { + "description": "Year", + "isAbout": [], + "name": "year", + "type": "integer", + "unit": "a", + "valueReference": [] + }, + { + "description": "Maximum allowed throughput power output capacity per process.", + "isAbout": [], + "name": "capacity_p_max", + "type": "float array", + "unit": "MW", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of hydrogen before being transported", + "isAbout": [], + "name": "conversion_factor_sec_hydrogen_orig", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of electricity demand for pipeline transport.", + "isAbout": [], + "name": "conversion_factor_sec_elec", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Commodity conversion factor of hydrogen after being transported", + "isAbout": [], + "name": "conversion_factor_sec_hydrogen", + "type": "float array", + "unit": "MWh/MWh", + "valueReference": [] + }, + { + "description": "Investment costs per unit capacity", + "isAbout": [], + "name": "cost_inv_p", + "type": "float array", + "unit": "\u20ac/MW", + "valueReference": [] + }, + { + "description": "Fixed operational costs per unit capacity", + "isAbout": [], + "name": "cost_fix_p", + "type": "float array", + "unit": "\u20ac/MW*a", + "valueReference": [] + }, + { + "description": "Lifetime in operation.", + "isAbout": [], + "name": "lifetime", + "type": "float array", + "unit": "a", + "valueReference": [] + }, + { + "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.", + "isAbout": [], + "name": "wacc", + "type": "text", + "unit": "%", + "valueReference": [] + }, + { + "description": "Bandwidth Type", + "isAbout": [], + "name": "bandwidth_type", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Version", + "isAbout": [], + "name": "version", + "type": "text", + "unit": null, + "valueReference": [] + }, + { + "description": "Method", + "isAbout": [], + "name": "method", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Source", + "isAbout": [], + "name": "source", + "type": "json", + "unit": null, + "valueReference": [] + }, + { + "description": "Comment", + "isAbout": [], + "name": "comment", + "type": "text", + "unit": null, + "valueReference": [] + } + ], + "foreignKeys": [], + "primaryKey": [ + "id" + ] + } + } + ], + "review": null, + "sources": [ + { + "description": null, + "licenses": [ + { + "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.", + "instruction": null, + "name": null, + "path": null, + "title": null + } + ], + "path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765", + "title": "Determining the lowest-cost hydrogen delivery mode" + }, + { + "description": null, + "licenses": [ + { + "attribution": "Copyright. Frontier Economics, 2021.", + "instruction": null, + "name": null, + "path": null, + "title": null + } + ], + "path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf", + "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines" + }, + { + "description": null, + "licenses": [ + { + "attribution": "Copyright. DVGW Bonn, 2023.", + "instruction": null, + "name": null, + "path": null, + "title": null + } + ], + "path": "https://www.dvgw.de/medien/dvgw/leistungen/publikationen/gasnetz-rueckgrat-h2-welt.pdf", + "title": "Das Gasnetz - R\u00fcckgrat der Wasserstoffwelt" + } + ], + "spatial": { + "extent": null, + "location": "global", + "resolution": null + }, + "subject": [ + { + "name": "hydrogen", + "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220" + }, + { + "name": "pipeline", + "path": "" + } + ], + "temporal": { + "referenceDate": null, + "timeseries": [ + { + "aggregationType": null, + "alignment": null, + "end": null, + "resolution": null, + "start": null + } + ] + }, + "title": "sedos_x2x_delivery_hydrogen_pipeline_retrofit_1" +} diff --git a/tests/test_data/test_collections/test_bandwidth/collection.json b/tests/test_data/test_collections/test_bandwidth/collection.json index 9cdb00d..8220b3b 100644 --- a/tests/test_data/test_collections/test_bandwidth/collection.json +++ b/tests/test_data/test_collections/test_bandwidth/collection.json @@ -1 +1,30 @@ -{"name": "https://databus.openenergyplatform.org/felixmaur/collections/test_bandwidth", "version": "v3", "artifacts": {"SEDOS_x2x_sector": {"x2x_delivery_hydrogen_pipeline_new_1": {"latest_version": "srd_range_draft", "multiple_types": false, "names": ["x2x_delivery_hydrogen_pipeline_new_1"], "subjects": ["hydrogen_pipeline"], "datatype": 0}, "x2x_delivery_hydrogen_pipeline_retrofit_1": {"latest_version": "srd_range_draft", "multiple_types": false, "names": ["x2x_delivery_hydrogen_pipeline_retrofit_1"], "subjects": ["hydrogen_pipeline"], "datatype": 0}}}} \ No newline at end of file +{ + "artifacts": { + "SEDOS_x2x_sector": { + "x2x_delivery_hydrogen_pipeline_new_1": { + "datatype": 0, + "latest_version": "srd_range_draft", + "multiple_types": false, + "names": [ + "x2x_delivery_hydrogen_pipeline_new_1" + ], + "subjects": [ + "hydrogen_pipeline" + ] + }, + "x2x_delivery_hydrogen_pipeline_retrofit_1": { + "datatype": 0, + "latest_version": "srd_range_draft", + "multiple_types": false, + "names": [ + "x2x_delivery_hydrogen_pipeline_retrofit_1" + ], + "subjects": [ + "hydrogen_pipeline" + ] + } + } + }, + "name": "https://databus.openenergyplatform.org/felixmaur/collections/test_bandwidth", + "version": "v3" +} diff --git a/tests/test_data/test_structures/emission_constraint_example.xlsx b/tests/test_data/test_structures/emission_constraint_example.xlsx new file mode 100644 index 0000000..3ce317e Binary files /dev/null and b/tests/test_data/test_structures/emission_constraint_example.xlsx differ diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index e568e4f..4cb904c 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -26,13 +26,13 @@ def test_process_of_artifact_with_multiple_subprocesses(): assert hasattr(onshore, "scalars") assert hasattr(onshore, "timeseries") - assert len(onshore.scalars.columns) == 12 + assert len(onshore.scalars.columns) == 7 assert len(onshore.scalars) == 50 assert onshore.timeseries["wind_speed", ("HE",)][0] == 1003 assert hasattr(offshore, "scalars") assert hasattr(offshore, "timeseries") - assert len(offshore.scalars.columns) == 12 + assert len(offshore.scalars.columns) == 7 assert len(offshore.scalars) == 50 assert offshore.timeseries["wind_speed", ("HE",)][0] == 10003 @@ -218,3 +218,25 @@ def test_bandwidth_unpacking(): pandas.Series([None, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]), check_names=False, ) + + +def test_return_only_relevant_columns(): + adapter = preprocessing.Adapter("test_return_only_relevant_columns") + + assert list(adapter.get_process("ind_other_boiler_hot_water_bio_1").scalars.columns.values) == [ + "region", + "year", + "availability_constant", + "conversion_factor_pri_biomass_stemwood", + "conversion_factor_iip_hot_water", + "cost_fix_p", + "cost_inv_p", + "lifetime", + "ef_pri_biomass_stemwood_emi_co2_f_ind", + "ef_pri_biomass_stemwood_emi_ch4_f_ind", + "ef_pri_biomass_stemwood_emi_n2o_f_ind", + "bandwidth_type", + "method", + "source", + "comment", + ] diff --git a/tests/test_structure.py b/tests/test_structure.py index 7df8ccd..9af63d1 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -19,3 +19,12 @@ def test_processes(): st = structure.Structure("modex_example") assert len(st.processes) == 4 assert st.processes["energy transformation unit"]["outputs"] == [["electricity", "heat"], "co2"] + + +def test_emission_constraint(): + st = structure.Structure("emission_constraint_example") + processes_raw = st.processes + assert len(st.processes) == 8 + expected_input = {'emi_n2o_f_ind', 'emi_co2_neg_air_dacc', 'emi_co2_f_ind', 'emi_ch4_f_ind'} + assert set(processes_raw['ind_constraint_co2eq']['inputs']) == expected_input +