Feature/emission constraint

data_adapter/preprocessing.py

@@ -287,6 +287,7 @@ def __unpack_bandwidths(self, df: pd.DataFrame) -> pd.DataFrame:
     @staticmethod
     def __filter_subprocess(df: pd.DataFrame, subprocess: str) -> pd.DataFrame:
         df = df[df["type"] == subprocess]
+        df = df.dropna(axis=1, how="all")
         return df.drop("type", axis=1)
 
     @staticmethod

data_adapter/structure.py

@@ -179,11 +179,42 @@ def get_nodes(nodes_raw):
             nodes += [node for node in nodes_raw_stripped.split(",") if node != ""]
             return nodes
 
+        def process_emission_constraint_inputs(processes_raw):
+            """If there's a row in the table where the process is `ind_constraint_co2eq`,
+              take all unique values in the `output` column that start with `emi` and
+              place them into the `input` field of that row.
+
+            refer: https://github.com/sedos-project/data_adapter/pull/39
+            """
+
+            ind_constraint_row = processes_raw[processes_raw["process"] == "ind_constraint_co2eq"]
+
+            if ind_constraint_row.empty:
+                return processes_raw
+
+            emi_outputs = []
+            for output in processes_raw["output"]:
+                parts = output.split(",")
+                for part in parts:
+                    part = part.strip()
+                    if part.startswith("emi_"):
+                        emi_outputs.append(part)
+
+            emi_outputs = list(set(emi_outputs))
+            emi_output_str = ",".join(emi_outputs)
+
+            if len(emi_outputs) == 0:
+                return processes_raw
+
+            processes_raw.loc[processes_raw["process"] == "ind_constraint_co2eq", "input"] = emi_output_str
+            return processes_raw
+
         processes_raw = pd.read_excel(
             io=self.structure_file,
             sheet_name=process_sheet,
             usecols=("process", "input", "output"),
         )
+
         wb = load_workbook(self.structure_file, read_only=True)
         if helper_sheet in wb.sheetnames:
             helpers_raw = pd.read_excel(
@@ -193,8 +224,10 @@ def get_nodes(nodes_raw):
             )
             processes_raw = pd.concat([processes_raw, helpers_raw])
         processes_raw = processes_raw.fillna("")
+        processes_raw = process_emission_constraint_inputs(processes_raw)
         check_character_convention(processes_raw, ["process"])
         processes = processes_raw.to_dict(orient="records")
+
         return {
             process["process"]: {"inputs": get_nodes(process["input"]), "outputs": get_nodes(process["output"])}
             for process in processes

tests/test_data/test_collections/test_bandwidth/SEDOS_x2x_sector/x2x_delivery_hydrogen_pipeline_new_1/srd_range_draft/x2x_delivery_hydrogen_pipeline_new_1.json

@@ -1 +1,263 @@
-{"id": null, "@id": null, "name": "x2x_delivery_hydrogen_pipeline_new_1", "title": "sedos_x2x_delivery_hydrogen_pipeline_new_1", "review": null, "context": {"contact": "g.mueller@fz-juelich.de", "grantNo": "03EI1040D", "homepage": "https://sedos-project.github.io/.github/", "sourceCode": "https://github.com/sedos-project", "documentation": "https://sedos-project.github.io/.github/", "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)", "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg"}, "sources": [{"path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765", "title": "Determining the lowest-cost hydrogen delivery mode", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.", "instruction": null}], "description": null}, {"path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf", "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines", "licenses": [{"name": null, "path": null, "title": null, "attribution": "Copyright. Frontier Economics, 2021.", "instruction": null}], "description": null}], "spatial": {"extent": null, "location": "global", "resolution": null}, "subject": [{"name": "hydrogen", "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220"}, {"name": "pipeline", "path": ""}], "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json", "_comment": {"null": "If not applicable use: null", "todo": "If a value is not yet available, use: todo", "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)", "units": "Use a space between numbers and units (100 m)", "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)", "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)", "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)", "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)"}, "keywords": ["SEDOS", "AP5", "X2X", "oedatamodel-parameter", "input_data"], "language": ["en-GB"], "licenses": [{"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode", "title": "Creative Commons Zero v1.0 Universal", "attribution": null, "instruction": "You are free: To Share, To Create, To Adapt"}], "temporal": {"timeseries": [{"end": null, "start": null, "alignment": null, "resolution": null, "aggregationType": null}], "referenceDate": null}, "resources": [{"name": "model_draft.x2x_delivery_hydrogen_pipeline_new_1", "path": null, "format": "PostgreSQL", "schema": {"fields": [{"name": "id", "type": "bigint", "unit": null, "isAbout": [], "description": "Unique identifier", "valueReference": []}, {"name": "region", "type": "text", "unit": null, "isAbout": [], "description": "Country or region", "valueReference": []}, {"name": "year", "type": "integer", "unit": "a", "isAbout": [], "description": "Year", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen_orig", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen before being transported", "valueReference": []}, {"name": "conversion_factor_sec_elec", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of electricity demand for pipeline transport.", "valueReference": []}, {"name": "conversion_factor_sec_hydrogen", "type": "float array", "unit": "MWh/MWh", "isAbout": [], "description": "Commodity conversion factor of hydrogen after being transported", "valueReference": []}, {"name": "cost_inv_p", "type": "float array", "unit": "\u20ac/MW", "isAbout": [], "description": "Investment costs per unit capacity", "valueReference": []}, {"name": "cost_fix_p", "type": "float array", "unit": "\u20ac/MW*a", "isAbout": [], "description": "Fixed operational costs per unit capacity", "valueReference": []}, {"name": "lifetime", "type": "float array", "unit": "a", "isAbout": [], "description": "Lifetime in operation.", "valueReference": []}, {"name": "wacc", "type": "text", "unit": "%", "isAbout": [], "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.", "valueReference": []}, {"name": "bandwidth_type", "type": "json", "unit": null, "isAbout": [], "description": "Bandwidth Type", "valueReference": []}, {"name": "version", "type": "text", "unit": null, "isAbout": [], "description": "Version", "valueReference": []}, {"name": "method", "type": "json", "unit": null, "isAbout": [], "description": "Method", "valueReference": []}, {"name": "source", "type": "json", "unit": null, "isAbout": [], "description": "Source", "valueReference": []}, {"name": "comment", "type": "text", "unit": null, "isAbout": [], "description": "Comment", "valueReference": []}], "primaryKey": ["id"], "foreignKeys": []}, "dialect": {"delimiter": ";", "decimalSeparator": "."}, "profile": "tabular-data-resource", "encoding": "UTF-8"}], "description": "Techno-economic parameter for newly built hydrogen pipelines in Germany.", "contributors": [{"date": "2024-04-08", "email": "g.mueller@fz-juelich.de", "title": "Gian M\u00fcller, FZJ", "object": "data and metadata", "comment": "Data for transport process of hydrogen by newly built pipeline in Germany."}], "metaMetadata": {"metadataLicense": {"name": "CC0-1.0", "path": "https://creativecommons.org/publicdomain/zero/1.0/", "title": "Creative Commons Zero v1.0 Universal"}, "metadataVersion": "OEP-1.5.2"}, "publicationDate": "2024-04-29"}
+{
+  "@context": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/develop/metadata/latest/context.json",
+  "@id": null,
+  "_comment": {
+    "dates": "Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss\u00b1hh)",
+    "languages": "Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
+    "licenses": "License name must follow the SPDX License List (https://spdx.org/licenses/)",
+    "metadata": "Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/oemetadata)",
+    "null": "If not applicable use: null",
+    "review": "Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/blob/master/data-review/manual/review_manual.md)",
+    "todo": "If a value is not yet available, use: todo",
+    "units": "Use a space between numbers and units (100 m)"
+  },
+  "context": {
+    "contact": "g.mueller@fz-juelich.de",
+    "documentation": "https://sedos-project.github.io/.github/",
+    "fundingAgency": "Bundesministerium f\u00fcr Wirtschaft und Klimaschutz (BMWK)",
+    "fundingAgencyLogo": "https://en.wikipedia.org/wiki/Federal_Ministry_for_Economic_Affairs_and_Climate_Action#/media/File:BMWi_Logo_2021.svg",
+    "grantNo": "03EI1040D",
+    "homepage": "https://sedos-project.github.io/.github/",
+    "sourceCode": "https://github.com/sedos-project"
+  },
+  "contributors": [
+    {
+      "comment": "Data for transport process of hydrogen by newly built pipeline in Germany.",
+      "date": "2024-04-08",
+      "email": "g.mueller@fz-juelich.de",
+      "object": "data and metadata",
+      "title": "Gian M\u00fcller, FZJ"
+    }
+  ],
+  "description": "Techno-economic parameter for newly built hydrogen pipelines in Germany.",
+  "id": null,
+  "keywords": [
+    "SEDOS",
+    "AP5",
+    "X2X",
+    "oedatamodel-parameter",
+    "input_data"
+  ],
+  "language": [
+    "en-GB"
+  ],
+  "licenses": [
+    {
+      "attribution": null,
+      "instruction": "You are free: To Share, To Create, To Adapt",
+      "name": "CC0-1.0",
+      "path": "https://creativecommons.org/publicdomain/zero/1.0/legalcode",
+      "title": "Creative Commons Zero v1.0 Universal"
+    }
+  ],
+  "metaMetadata": {
+    "metadataLicense": {
+      "name": "CC0-1.0",
+      "path": "https://creativecommons.org/publicdomain/zero/1.0/",
+      "title": "Creative Commons Zero v1.0 Universal"
+    },
+    "metadataVersion": "OEP-1.5.2"
+  },
+  "name": "x2x_delivery_hydrogen_pipeline_new_1",
+  "publicationDate": "2024-04-29",
+  "resources": [
+    {
+      "dialect": {
+        "decimalSeparator": ".",
+        "delimiter": ";"
+      },
+      "encoding": "UTF-8",
+      "format": "PostgreSQL",
+      "name": "model_draft.x2x_delivery_hydrogen_pipeline_new_1",
+      "path": null,
+      "profile": "tabular-data-resource",
+      "schema": {
+        "fields": [
+          {
+            "description": "Unique identifier",
+            "isAbout": [],
+            "name": "id",
+            "type": "bigint",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Country or region",
+            "isAbout": [],
+            "name": "region",
+            "type": "text",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Year",
+            "isAbout": [],
+            "name": "year",
+            "type": "integer",
+            "unit": "a",
+            "valueReference": []
+          },
+          {
+            "description": "Commodity conversion factor of hydrogen before being transported",
+            "isAbout": [],
+            "name": "conversion_factor_sec_hydrogen_orig",
+            "type": "float array",
+            "unit": "MWh/MWh",
+            "valueReference": []
+          },
+          {
+            "description": "Commodity conversion factor of electricity demand for pipeline transport.",
+            "isAbout": [],
+            "name": "conversion_factor_sec_elec",
+            "type": "float array",
+            "unit": "MWh/MWh",
+            "valueReference": []
+          },
+          {
+            "description": "Commodity conversion factor of hydrogen after being transported",
+            "isAbout": [],
+            "name": "conversion_factor_sec_hydrogen",
+            "type": "float array",
+            "unit": "MWh/MWh",
+            "valueReference": []
+          },
+          {
+            "description": "Investment costs per unit capacity",
+            "isAbout": [],
+            "name": "cost_inv_p",
+            "type": "float array",
+            "unit": "\u20ac/MW",
+            "valueReference": []
+          },
+          {
+            "description": "Fixed operational costs per unit capacity",
+            "isAbout": [],
+            "name": "cost_fix_p",
+            "type": "float array",
+            "unit": "\u20ac/MW*a",
+            "valueReference": []
+          },
+          {
+            "description": "Lifetime in operation.",
+            "isAbout": [],
+            "name": "lifetime",
+            "type": "float array",
+            "unit": "a",
+            "valueReference": []
+          },
+          {
+            "description": "Percentage of costs for capital after taxes. Used to calculate annuity factor for investment costs.",
+            "isAbout": [],
+            "name": "wacc",
+            "type": "text",
+            "unit": "%",
+            "valueReference": []
+          },
+          {
+            "description": "Bandwidth Type",
+            "isAbout": [],
+            "name": "bandwidth_type",
+            "type": "json",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Version",
+            "isAbout": [],
+            "name": "version",
+            "type": "text",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Method",
+            "isAbout": [],
+            "name": "method",
+            "type": "json",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Source",
+            "isAbout": [],
+            "name": "source",
+            "type": "json",
+            "unit": null,
+            "valueReference": []
+          },
+          {
+            "description": "Comment",
+            "isAbout": [],
+            "name": "comment",
+            "type": "text",
+            "unit": null,
+            "valueReference": []
+          }
+        ],
+        "foreignKeys": [],
+        "primaryKey": [
+          "id"
+        ]
+      }
+    }
+  ],
+  "review": null,
+  "sources": [
+    {
+      "description": null,
+      "licenses": [
+        {
+          "attribution": "Copyright \u00a9 2006 International Association for Hydrogen Energy. Published by Elsevier Ltd. All rights reserved.",
+          "instruction": null,
+          "name": null,
+          "path": null,
+          "title": null
+        }
+      ],
+      "path": "https://linkinghub.elsevier.com/retrieve/pii/S0360319906001765",
+      "title": "Determining the lowest-cost hydrogen delivery mode"
+    },
+    {
+      "description": null,
+      "licenses": [
+        {
+          "attribution": "Copyright. Frontier Economics, 2021.",
+          "instruction": null,
+          "name": null,
+          "path": null,
+          "title": null
+        }
+      ],
+      "path": "https://www.frontier-economics.com/media/ovdnwcyx/frontier-ptx-business-cases-studie_teil-2-pipelinetransport.pdf",
+      "title": "Teil 2: Pipeline Studie Kosten von gr\u00fcnem Wasserstoff Import via Pipelines"
+    }
+  ],
+  "spatial": {
+    "extent": null,
+    "location": "global",
+    "resolution": null
+  },
+  "subject": [
+    {
+      "name": "hydrogen",
+      "path": "http://openenergy-platform.org/ontology/oeo/OEO_00000220"
+    },
+    {
+      "name": "pipeline",
+      "path": ""
+    }
+  ],
+  "temporal": {
+    "referenceDate": null,
+    "timeseries": [
+      {
+        "aggregationType": null,
+        "alignment": null,
+        "end": null,
+        "resolution": null,
+        "start": null
+      }
+    ]
+  },
+  "title": "sedos_x2x_delivery_hydrogen_pipeline_new_1"
+}