From d60c70920a665ebd2136c6933d84536119129cea Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:39:22 -0600 Subject: [PATCH 1/5] Create notes.md --- docs/notes.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/notes.md diff --git a/docs/notes.md b/docs/notes.md new file mode 100644 index 00000000..09442ab9 --- /dev/null +++ b/docs/notes.md @@ -0,0 +1,23 @@ +# Notes on a "NextGen Universal Package" + +There's been some discussion in the community about the combinatorial effects of opening up the possibility to build ever-more-complicated systems while making it more difficult to resproduce results in the community, or to share results for integration into our collective understanding of the world. + +The first step to building that collective understanding is based on having the same interpretation and categorization of the data we have and will produce. + +Firstly there must be a heirarchical underestanding of how we represent data and configurations. Note that these aren't 1:1 with the data and configurations, these are for the heirarchical framing of how those are organized. + +The identified components are as follows: + +- _Data_ + - Hydrofabric + - Underlying physical understanding of the domain + - Forcings + - Input data over a temporal and spacial domain +- _Configuration_ + - Realization of model allocation + - Models (version of BMI-compatible model) + - Model configurations + +Each Instance of the combination of these things is hereafter called a RUN and the grouping of all the aforementioned items necessitates a standardized RUN CONFIG. + +A preliminary Specification for NextGen RUN CONFIGs has been proposed. From 7a05e5139811f700128beb499f4c773e26f5c965 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:41:05 -0600 Subject: [PATCH 2/5] Package Proposal I've added the necessary framework for semi-reproducible cataloging of a RUN. This is a relatively flexible item, since no automation has been produced yet. Eventually existing tooling will make this spec more difficult to change. --- Example.json | 47 ++++++++++++++++++++ README.md | 111 ++++++++++++++++++++++------------------------- Specification.md | 26 +++++++++++ 3 files changed, 126 insertions(+), 58 deletions(-) create mode 100644 Example.json create mode 100644 Specification.md diff --git a/Example.json b/Example.json new file mode 100644 index 00000000..955d77cb --- /dev/null +++ b/Example.json @@ -0,0 +1,47 @@ +{ + "RUN_CONFIG": { + "description": "Run Config for Hurricane Zach", + "Model": { + "realization": { + "Name": "CFE_SLOTH_realization", + "Type": "filesystem", + "Path": "/ngen/realization.json", + "Hash": "d2c6fbda93c134de495d69745fae11087784d2aa" + }, + "configuration": { + "Name": "CFE_SLOTH_ini", + "Type": "filesystem", + "Path": "/ngen/cfe_sloth.ini", + "Hash": "ab0c3dff59c4b282b172b90128159fda3386d012" + } + }, + "Forcings": { + "inputs": { + "Name": "Hurricane Zach", + "Type": "bucket", + "Path": "s3://awi-ciroh-ngen-data/AWI_001/forcings/", + "Hash": "220fff8bdd3b85f23d93e73b4bc7e3bc2c7c0f35" + }, + "Hydrofabric": { + "catchment": { + "Name": "Catchment(s) File v1.0", + "Type": "bucket", + "Path": "s3://awi-ciroh-ngen-data/AWI_001/catchments.geojson", + "Hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + "nexus": { + "Name": "Nexus File v1.0", + "Type": "bucket", + "Path": "s3://awi-ciroh-ngen-data/AWI_001/nexus.geojson", + "Hash": "cae054f62f697080d822fea9c7d9c268be8b7ac9" + }, + "crosswalk": { + "Name": "Crosswalk File v1.0", + "Type": "bucket", + "Path": "s3://awi-ciroh-ngen-data/AWI_001/crosswalk.geojson", + "Hash": "4c39964d1e30779f9992d3c00e94a39952cb102a" + } + } + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index a9158499..f422e3c7 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,56 @@ -# CIROH Cloud Terraform Configuration +###Reproducing -This Terraform configuration is used to build and manage resources on the CIROH cloud, including an EC2 instance running a containerized model. - -## Docker Build - -To build the Dockerfile, run the following command: -```bash -docker buildx build -f /docker/Dockerfile --platform=linux/amd64 -t $CONTAINER_NAME . +Get the files from the appropriate bucket & RUN_CONFIG ``` -This is now done by Github Actions automatically, on push to main branch or when Pull Request is created to a branch - `main`. - -## Terraform validate - -- GitHub Actions will run Terraform init and validate every time a push or pull request is made to a branch - `main`. - -** Note that the terraform init command is also included in the workflow. This command initializes the Terraform working directory and downloads the necessary providers and modules. It is required before running terraform validate to ensure that all dependencies are available. - -## Terraform Usage - -To run the Terraform configuration, follow these steps: - -1. Generate a plan file by running terraform plan -o plan.file. -2. Apply the plan file by running terraform apply plan.file and fill out the Terraform variables. - -## Terraform Variables -The following variables can be customized: - -`aws_region`: The preferred region in which to launch EC2 instances. Defaults to us-east-1. - -`nameprefix`: Prefix to use for some resource names to avoid duplicates. Default value is "Cloud-Example". - -`name_tag`: Value of the Name tag for the EC2 instance. Default value is "Cloud-Example-Terraform". - -`project_tag`: Value of the Project tag for the EC2 instance. Default value is "Cloud-Example". - -`availability_zone`: Availability zone to use. Default value is "us-east-1a". - -`instance_type`: EC2 Instance Type. Larger instance is "c5n.18xlarge". Default value is "t3.medium". +wget --no-parent https://awi-ciroh-ngen-data.s3.us-east-2.amazonaws.com/AWI_001/AWI_03W_113060_001.tar.gz . -`use_efa`: Attach EFA Network. Default value is "true". - -`key_name`: The name of the ssh key-pair used to access the EC2 instances. - -`container_name`: The name of the containerized model to run. Default value is "zwills/dmod_ngen_slim". - -`ngen_catchment_file`: The path of the catchment file, /mnt is the S3 mount default; then path to examples in the ngen repo. - -`ngen_nexus_file`: The path of the nexus file, /mnt is the S3 mount default; examples in the ngen repo. - -`ngen_realization_file`: The path of the ngen realization file, /mnt is the S3 mount default; examples in the ngen repo. - -`allowed_ssh_cidr`: Public IP address/range allowed for SSH access. - -`bucket_name`: S3 Bucket Name for AWS bucket to mount (at /mnt) for data. - -`public_key`: Contents of the SSH public key to be used for authentication. - -`managed_policies`: The attached IAM policies granting machine permissions. Default value is ["arn:aws:iam::aws:policy/AmazonEC2FullAccess", "arn:aws:iam::aws:policy/AmazonS3FullAccess", "arn:aws:iam::aws:policy/AmazonFSxFullAccess"]. - -`ami_id`: The random ID used for AMIs. Default value is "unknown value". +tar -xvf AWI_03W_113060_001.tar.gz +``` +Then we can confirm file location and integrity against the example JSON file. +``` +{ + "RUN_CONFIG": { + "description": "Run Config for AWI_03W_113060_001", + "Model": { + "realization": { + "Name": "AWI_simplified_realization", + "Type": "filesystem", + "Path": "AWI_03W_113060_001/config/awi_simplified_realization.json", + "Hash": "792554dcf48b61120cfc648cc6711d2b5e61d321" + }, + "configuration": { + "Name": "CFE_SLOTH_ini", + "Type": "filesystem", + "Path": "AWI_03W_113060_001/config/awi_config.ini", + "Hash": "e8283864026040ce1ce5a7dca79b9f4f04744b47" + } + }, + "Forcings": { + "inputs": { + "Name": "Hurricane Zach", + "Type": "filesystem", + "Path": "AWI_03W_113060_001/forcings", + "Hash": "da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + "Hydrofabric": { + "catchment": { + "Name": "Catchment(s) File v1.0", + "Type": "bucket", + "Path": "AWI_03W_113060_001/config/catchment_data.geojson", + "Hash": "880feb145f254976600bd8968ef730105de6cbee" + }, + "nexus": { + "Name": "Nexus File v1.0", + "Type": "bucket", + "Path": "AWI_03W_113060_001/config/nexus_data.geojson", + "Hash": "86a029a15e7cf67bc69f2390038a74b69b09af04" + }, + } + } + } +} +``` +Note that to recreate the sums for the focings file, I simply summed the files then piped the output to its own sum. +``` +shasum -a 256 AWI_03W_113060_001/forcings/ | shasum +``` diff --git a/Specification.md b/Specification.md new file mode 100644 index 00000000..e3fa1ce9 --- /dev/null +++ b/Specification.md @@ -0,0 +1,26 @@ +--- CONFIG START --- +### Model +| Name of Model files | type | PATH to .json file | HASH of file | +| ----------- | ----------- | ----------- | ----------- | +| CFE_SLOTH_realization | filesystem | /ngen/realization.json | d2c6fbda93c134de495d69745fae11087784d2aa | +| CFE_SLOTH_ini | filesystem | /ngen/cfe_sloth.ini | ab0c3dff59c4b282b172b90128159fda3386d012 | +### Forcings +| Name of Forcings | type | PATH to forcings file(s) | HASH of file(s) | +| ----------- | ----------- | ----------- | ----------- | +| Hurricane Zach | bucket | s3://awi-ciroh-ngen-data/AWI_001/forcings/ | 220fff8bdd3b85f23d93e73b4bc7e3bc2c7c0f35 | +### Hydrofabric +| Name of Hydrofabric | type | PATH to .json file | HASH of file | +| ----------- | ----------- | ----------- | ----------- | +| Catchment(s) File | bucket | s3://awi-ciroh-ngen-data/AWI_001/catchments.geojson | da39a3ee5e6b4b0d3255bfef95601890afd80709 | +| Nexus File | bucket | s3://awi-ciroh-ngen-data/AWI_001/nexus.geojson | cae054f62f697080d822fea9c7d9c268be8b7ac9 | +| Crosswalk File | bucket | s3://awi-ciroh-ngen-data/AWI_001/crosswalk.geojson | 4c39964d1e30779f9992d3c00e94a39952cb102a | + +All values within each section are defined for the run to evaluate when a run is unique and the change that makes it so. For example changing the hash of any of the fields for the referenced files should change the RUN CONFIG into something new. + +Note: the Realization, Catchment, Nexus and other model-required files must be searching by name. + +| Valid | Not Valid | +| ----- | --------- | +| hurricane_marty_realization.json | hurricane_marty.json | +| Houston_catchments.geojson | Houston.geojson | +| Nexus_2012_flood.geojson | 2012_flood.geojson | \ No newline at end of file From 7afe3f0a925f4b1c0e2433fe3acb80a825067543 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:41:29 -0600 Subject: [PATCH 3/5] Rename Example.json to docs/Example.json --- Example.json => docs/Example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Example.json => docs/Example.json (99%) diff --git a/Example.json b/docs/Example.json similarity index 99% rename from Example.json rename to docs/Example.json index 955d77cb..4978fbb7 100644 --- a/Example.json +++ b/docs/Example.json @@ -44,4 +44,4 @@ } } } -} \ No newline at end of file +} From 0d296b4b3386d310b96cad998948891f1020afad Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:41:47 -0600 Subject: [PATCH 4/5] Rename README.md to docs/README.md --- README.md => docs/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README.md => docs/README.md (100%) diff --git a/README.md b/docs/README.md similarity index 100% rename from README.md rename to docs/README.md From 635dafa7696fb4854cfff27de67c1d470b7f0480 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:41:58 -0600 Subject: [PATCH 5/5] Rename Specification.md to docs/Specification.md --- Specification.md => docs/Specification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Specification.md => docs/Specification.md (96%) diff --git a/Specification.md b/docs/Specification.md similarity index 96% rename from Specification.md rename to docs/Specification.md index e3fa1ce9..32fb2a05 100644 --- a/Specification.md +++ b/docs/Specification.md @@ -23,4 +23,4 @@ Note: the Realization, Catchment, Nexus and other model-required files must be s | ----- | --------- | | hurricane_marty_realization.json | hurricane_marty.json | | Houston_catchments.geojson | Houston.geojson | -| Nexus_2012_flood.geojson | 2012_flood.geojson | \ No newline at end of file +| Nexus_2012_flood.geojson | 2012_flood.geojson |