From 5ff8c13bae230c1c2c831cd584858cc8beb1b625 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 11:58:58 -0500 Subject: [PATCH 01/10] creates service lambda --- .gitignore | 3 + main.go | 12 ++ scripts/compute-node-service.sh | 17 +++ terraform/compute-node-service/README.md | 1 + .../compute-node-service.py | 107 ++++++++++++++++ terraform/compute-node-service/lambda.tf | 121 ++++++++++++++++++ terraform/compute-node-service/main.tf | 4 + terraform/compute-node-service/outputs.tf | 5 + 8 files changed, 270 insertions(+) create mode 100755 scripts/compute-node-service.sh create mode 100644 terraform/compute-node-service/README.md create mode 100644 terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py create mode 100644 terraform/compute-node-service/lambda.tf create mode 100644 terraform/compute-node-service/main.tf create mode 100644 terraform/compute-node-service/outputs.tf diff --git a/.gitignore b/.gitignore index ad41797..41feafe 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ terraform/status-service/.terraform/providers/registry.terraform.io/hashicorp/ar terraform/status-service/.terraform/providers/registry.terraform.io/hashicorp/aws/5.36.0/linux_arm64/terraform-provider-aws_v5.36.0_x5 terraform/status-service/.terraform/providers/registry.terraform.io/hashicorp/random/3.6.0/linux_arm64/terraform-provider-random_v3.6.0_x5 terraform/status-service/terraform.tfstate.backup +terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/archive/2.4.2/linux_arm64/terraform-provider-archive_v2.4.2_x5 +terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/aws/5.38.0/linux_arm64/terraform-provider-aws_v5.38.0_x5 +terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/random/3.6.0/linux_arm64/terraform-provider-random_v3.6.0_x5 diff --git a/main.go b/main.go index 3876b3d..6e992ad 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ var TerraformAppStateDirectory = "/service/terraform/application-state" var TerraformGatewayDirectory = "/service/terraform/internet-gateway" var TerraformApplicationDirectory = "/service/terraform/application-wrapper" var TerraformStatusServiceDirectory = "/service/terraform/status-service" +var TerraformComputeServiceDirectory = "/service/terraform/compute-node-service" func main() { cmdPtr := flag.String("cmd", "plan", "command to execute") @@ -83,5 +84,16 @@ func main() { } } + // compute node service + if *cmdPtr == "create-compute-node" || *cmdPtr == "destroy-compute-node" { + cmd := exec.Command("/bin/sh", "./scripts/compute-node-service.sh", TerraformComputeServiceDirectory, *cmdPtr) + out, err := cmd.Output() + output := string(out) + fmt.Println(output) + if err != nil { + log.Fatalf("error %s", err.Error()) + } + } + log.Println("done") } diff --git a/scripts/compute-node-service.sh b/scripts/compute-node-service.sh new file mode 100755 index 0000000..284405b --- /dev/null +++ b/scripts/compute-node-service.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +cd $1 + +if [ $2 = "create-compute-node" ]; then + echo "creating ..." + # terraform validate + echo "running init ..." + export TF_LOG_PATH="error.log" + export TF_LOG=TRACE + terraform init > init.log + terraform plan -out=tfplan > plan.log + terraform apply tfplan +else + echo "deleting ..." + terraform apply -destroy -auto-approve +fi \ No newline at end of file diff --git a/terraform/compute-node-service/README.md b/terraform/compute-node-service/README.md new file mode 100644 index 0000000..1d4ab87 --- /dev/null +++ b/terraform/compute-node-service/README.md @@ -0,0 +1 @@ +Compute node service \ No newline at end of file diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py new file mode 100644 index 0000000..ebe0c56 --- /dev/null +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -0,0 +1,107 @@ +from boto3 import client as boto3_client +import json +import base64 +import os + +def lambda_handler(event, context): + + print(event) + + if event['isBase64Encoded'] == True: + body = base64.b64decode(event['body']).decode('utf-8') + event['body'] = body + event['isBase64Encoded'] = False + json_body = json.loads(event['body']) + print(json_body) + + # start Fargate task + if cluster_name != "": + print("Starting Fargate task") + response = ecs_client.run_task( + cluster = cluster_name, + launchType = 'FARGATE', + taskDefinition=task_definition_name, + count = 1, + platformVersion='LATEST', + networkConfiguration={ + 'awsvpcConfiguration': { + 'subnets': subnet_ids.split(","), + 'assignPublicIp': 'ENABLED', + 'securityGroups': [security_group] + } + }, + overrides={ + 'containerOverrides': [ + { + 'name': container_name, + 'environment': [ + { + 'name': 'INTEGRATION_ID', + 'value': integration_id + }, + { + 'name': 'BASE_DIR', + 'value': '/mnt/efs' + }, + { + 'name': 'TASK_DEFINITION_NAME_POST', + 'value': task_definition_name_post + }, + { + 'name': 'CONTAINER_NAME_POST', + 'value': container_name_post + }, + { + 'name': 'PENNSIEVE_API_KEY', + 'value': api_key + }, + { + 'name': 'PENNSIEVE_API_SECRET', + 'value': api_secret + }, + { + 'name': 'PENNSIEVE_API_HOST', + 'value': pennsieve_host + }, + { + 'name': 'PENNSIEVE_API_HOST2', + 'value': pennsieve_host2 + }, + { + 'name': 'PENNSIEVE_AGENT_HOME', + 'value': pennieve_agent_home + }, + { + 'name': 'PENNSIEVE_UPLOAD_BUCKET', + 'value': pennsieve_upload_bucket + }, + { + 'name': 'CLUSTER_NAME', + 'value': cluster_name + }, + { + 'name': 'SECURITY_GROUP_ID', + 'value': security_group + }, + { + 'name': 'SUBNET_IDS', + 'value': subnet_ids + }, + { + 'name': 'SESSION_TOKEN', + 'value': session_token + }, + { + 'name': 'ENVIRONMENT', + 'value': environment + }, + + ], + }, + ], + }) + + return { + 'statusCode': 202, + 'body': json.dumps(str('Compute node creation initiated')) + } \ No newline at end of file diff --git a/terraform/compute-node-service/lambda.tf b/terraform/compute-node-service/lambda.tf new file mode 100644 index 0000000..5f09fe4 --- /dev/null +++ b/terraform/compute-node-service/lambda.tf @@ -0,0 +1,121 @@ +// Application Gateway Lambda +resource "aws_lambda_function" "compute_node_service" { + function_name = "compute-node-service-${random_uuid.val.id}" + role = aws_iam_role.iam_for_lambda.arn + handler = "compute-node-service.lambda_handler" # module is name of python file: application + description = "Compute Node Service" + + s3_bucket = aws_s3_bucket.lambda_bucket.id + s3_key = aws_s3_object.compute_node_service_lambda.key + + source_code_hash = data.archive_file.compute_node_service_lambda.output_base64sha256 + + runtime = "python3.12" + timeout = 60 + + environment { + variables = { + # REGION = "us-east-1" + # CLUSTER_NAME = aws_ecs_cluster.pipeline_cluster.name + # TASK_DEFINITION_NAME = aws_ecs_task_definition.pipeline.family + # CONTAINER_NAME = aws_ecs_task_definition.pipeline.family # currently same as name of task definition + # SUBNET_IDS = local.subnet_ids + # SECURITY_GROUP_ID = aws_default_security_group.default.id + } + } +} + +resource "aws_cloudwatch_log_group" "compute_node_service-lambda" { + name = "/aws/lambda/${aws_lambda_function.compute_node_service.function_name}" + + retention_in_days = 30 +} + +resource "aws_lambda_function_url" "compute_node_service" { + function_name = aws_lambda_function.compute_node_service.function_name + authorization_type = "NONE" +} + +# IAM +// Lambda gateway function +// allow lambda to access resources in your AWS account +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "iam_for_lambda" { + name = "iam_for_lambda-${random_uuid.val.id}" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "compute_lambda_policy" { + role = aws_iam_role.iam_for_lambda.name + policy_arn = aws_iam_policy.lambda_iam_policy.arn +} + +resource "aws_iam_policy" "lambda_iam_policy" { + name = "lambda-iam-policy-${random_uuid.val.id}" + path = "/" + policy = data.aws_iam_policy_document.iam_policy_document_compute.json +} + +# data +// creates an archive and uploads to s3 bucket +data "archive_file" "compute_node_service_lambda" { + type = "zip" + + source_dir = "${path.module}/compute-node-service-lambda" + output_path = "${path.module}/compute-node-service-lambda.zip" +} + +// provides an s3 object resource +resource "aws_s3_object" "compute_node_service_lambda" { + bucket = aws_s3_bucket.lambda_bucket.id + + key = "compute-node-service-lambda.zip" + source = data.archive_file.compute_node_service_lambda.output_path + + etag = filemd5(data.archive_file.compute_node_service_lambda.output_path) +} + +// policy document - compute service lambda +data "aws_iam_policy_document" "iam_policy_document_compute" { + statement { + sid = "CloudwatchPermissions" + effect = "Allow" + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = ["*"] + } +} + +// S3 bucket +resource "aws_s3_bucket" "lambda_bucket" { + bucket = random_uuid.val.id +} + +resource "aws_s3_bucket_ownership_controls" "lambda_bucket" { + bucket = aws_s3_bucket.lambda_bucket.id + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_acl" "bucket_acl" { + depends_on = [aws_s3_bucket_ownership_controls.lambda_bucket] + + bucket = aws_s3_bucket.lambda_bucket.id + acl = "private" +} \ No newline at end of file diff --git a/terraform/compute-node-service/main.tf b/terraform/compute-node-service/main.tf new file mode 100644 index 0000000..fa0e639 --- /dev/null +++ b/terraform/compute-node-service/main.tf @@ -0,0 +1,4 @@ +provider "aws" {} + +resource "random_uuid" "val" { +} \ No newline at end of file diff --git a/terraform/compute-node-service/outputs.tf b/terraform/compute-node-service/outputs.tf new file mode 100644 index 0000000..d0b8783 --- /dev/null +++ b/terraform/compute-node-service/outputs.tf @@ -0,0 +1,5 @@ +output "lambda_function_url" { + description = "Lambda URL" + + value = aws_lambda_function_url.compute_node_service.function_url +} \ No newline at end of file From 4677beb5dd9b971cc2d82b5267126f82455d9789 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 11:59:31 -0500 Subject: [PATCH 02/10] adds to .gitignore and updates Makefile --- .gitignore | 3 +++ Makefile | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 41feafe..5423464 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ terraform/status-service/terraform.tfstate.backup terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/archive/2.4.2/linux_arm64/terraform-provider-archive_v2.4.2_x5 terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/aws/5.38.0/linux_arm64/terraform-provider-aws_v5.38.0_x5 terraform/compute-node-service/.terraform/providers/registry.terraform.io/hashicorp/random/3.6.0/linux_arm64/terraform-provider-random_v3.6.0_x5 +terraform/compute-node-service/.terraform.lock.hcl +terraform/compute-node-service/terraform.tfstate.backup +terraform/compute-node-service/tfplan diff --git a/Makefile b/Makefile index 7a12f2e..9139570 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,13 @@ create-status-service: docker-compose run app-deploy -cmd create-status-service destroy-status-service: - docker-compose run app-deploy -cmd destroy-status-service + docker-compose run app-deploy -cmd destroy-status-service + +create-compute-node: + docker-compose run app-deploy -cmd create-compute-node + +destroy-compute-node: + docker-compose run app-deploy -cmd destroy-compute-node deploy: aws ecr get-login-password --profile ${AWS_PROFILE} --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin ${ACCOUNT}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com From 092fac8df41e8c41ca458fdc60e487b26f94e3d0 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 13:01:17 -0500 Subject: [PATCH 03/10] adds cluster and ECR repo --- terraform/compute-node-service/ecr.tf | 8 ++++ terraform/compute-node-service/ecs.tf | 59 ++++++++++++++++++++++++ terraform/compute-node-service/lambda.tf | 4 +- 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 terraform/compute-node-service/ecr.tf create mode 100644 terraform/compute-node-service/ecs.tf diff --git a/terraform/compute-node-service/ecr.tf b/terraform/compute-node-service/ecr.tf new file mode 100644 index 0000000..3174cd1 --- /dev/null +++ b/terraform/compute-node-service/ecr.tf @@ -0,0 +1,8 @@ +resource "aws_ecr_repository" "app" { + name = "deploy-app-${random_uuid.val.id}" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = false # consider implications of setting to true + } +} \ No newline at end of file diff --git a/terraform/compute-node-service/ecs.tf b/terraform/compute-node-service/ecs.tf new file mode 100644 index 0000000..5b11cf0 --- /dev/null +++ b/terraform/compute-node-service/ecs.tf @@ -0,0 +1,59 @@ +// ECS Cluster +resource "aws_kms_key" "ecs_cluster" { + description = "ecs_cluster_kms_key" + deletion_window_in_days = 7 +} + +resource "aws_cloudwatch_log_group" "ecs_cluster" { + name = "ecs-cluster-log-${random_uuid.val.id}" +} + +resource "aws_ecs_cluster" "pipeline_cluster" { + name = "pipeline-cluster-${random_uuid.val.id}" + + configuration { + execute_command_configuration { + kms_key_id = aws_kms_key.ecs_cluster.arn + logging = "OVERRIDE" + + log_configuration { + cloud_watch_encryption_enabled = true + cloud_watch_log_group_name = aws_cloudwatch_log_group.ecs_cluster.name + } + } + } +} + +# // ECS Task definition +# resource "aws_ecs_task_definition" "pipeline" { +# family = "pipeline-${random_uuid.val.id}" +# requires_compatibilities = ["FARGATE"] +# network_mode = "awsvpc" +# cpu = 2148 +# memory = 4096 +# task_role_arn = aws_iam_role.task_role_for_ecs_task.arn +# execution_role_arn = aws_iam_role.execution_role_for_ecs_task.arn + +# container_definitions = jsonencode([ +# { +# name = "pipeline-${random_uuid.val.id}" +# image = aws_ecr_repository.app.repository_url +# essential = true +# portMappings = [ +# { +# containerPort = 8081 +# hostPort = 8081 +# } +# ] +# logConfiguration = { +# logDriver = "awslogs" +# options = { +# awslogs-group = "/ecs/pipeline/${random_uuid.val.id}" +# awslogs-region = var.region +# awslogs-stream-prefix = "ecs" +# awslogs-create-group = "true" +# } +# } +# } +# ]) +# } \ No newline at end of file diff --git a/terraform/compute-node-service/lambda.tf b/terraform/compute-node-service/lambda.tf index 5f09fe4..cc76c6a 100644 --- a/terraform/compute-node-service/lambda.tf +++ b/terraform/compute-node-service/lambda.tf @@ -15,8 +15,8 @@ resource "aws_lambda_function" "compute_node_service" { environment { variables = { - # REGION = "us-east-1" - # CLUSTER_NAME = aws_ecs_cluster.pipeline_cluster.name + REGION = "us-east-1" + CLUSTER_NAME = aws_ecs_cluster.pipeline_cluster.name # TASK_DEFINITION_NAME = aws_ecs_task_definition.pipeline.family # CONTAINER_NAME = aws_ecs_task_definition.pipeline.family # currently same as name of task definition # SUBNET_IDS = local.subnet_ids From 4314ee8405c012532c7aaefa11490f054203709d Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 13:12:54 -0500 Subject: [PATCH 04/10] adds task definition --- terraform/compute-node-service/ecs.tf | 64 ++++++++--------- terraform/compute-node-service/iam.tf | 84 +++++++++++++++++++++++ terraform/compute-node-service/lambda.tf | 4 +- terraform/compute-node-service/outputs.tf | 6 ++ 4 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 terraform/compute-node-service/iam.tf diff --git a/terraform/compute-node-service/ecs.tf b/terraform/compute-node-service/ecs.tf index 5b11cf0..5b5b633 100644 --- a/terraform/compute-node-service/ecs.tf +++ b/terraform/compute-node-service/ecs.tf @@ -24,36 +24,36 @@ resource "aws_ecs_cluster" "pipeline_cluster" { } } -# // ECS Task definition -# resource "aws_ecs_task_definition" "pipeline" { -# family = "pipeline-${random_uuid.val.id}" -# requires_compatibilities = ["FARGATE"] -# network_mode = "awsvpc" -# cpu = 2148 -# memory = 4096 -# task_role_arn = aws_iam_role.task_role_for_ecs_task.arn -# execution_role_arn = aws_iam_role.execution_role_for_ecs_task.arn +// ECS Task definition +resource "aws_ecs_task_definition" "pipeline" { + family = "pipeline-${random_uuid.val.id}" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = 2048 + memory = 4096 + task_role_arn = aws_iam_role.task_role_for_ecs_task.arn + execution_role_arn = aws_iam_role.execution_role_for_ecs_task.arn -# container_definitions = jsonencode([ -# { -# name = "pipeline-${random_uuid.val.id}" -# image = aws_ecr_repository.app.repository_url -# essential = true -# portMappings = [ -# { -# containerPort = 8081 -# hostPort = 8081 -# } -# ] -# logConfiguration = { -# logDriver = "awslogs" -# options = { -# awslogs-group = "/ecs/pipeline/${random_uuid.val.id}" -# awslogs-region = var.region -# awslogs-stream-prefix = "ecs" -# awslogs-create-group = "true" -# } -# } -# } -# ]) -# } \ No newline at end of file + container_definitions = jsonencode([ + { + name = "pipeline-${random_uuid.val.id}" + image = aws_ecr_repository.app.repository_url + essential = true + portMappings = [ + { + containerPort = 8081 + hostPort = 8081 + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = "/ecs/pipeline/${random_uuid.val.id}" + awslogs-region = "us-east-1" + awslogs-stream-prefix = "ecs" + awslogs-create-group = "true" + } + } + } + ]) +} \ No newline at end of file diff --git a/terraform/compute-node-service/iam.tf b/terraform/compute-node-service/iam.tf new file mode 100644 index 0000000..3417b10 --- /dev/null +++ b/terraform/compute-node-service/iam.tf @@ -0,0 +1,84 @@ +// ECS task IAM role +resource "aws_iam_role" "task_role_for_ecs_task" { + name = "task_role_for_ecs_task-${random_uuid.val.id}" + assume_role_policy = data.aws_iam_policy_document.ecs_task_role_assume_role.json + managed_policy_arns = [aws_iam_policy.ecs_run_task.arn] +} + +resource "aws_iam_policy" "ecs_run_task" { + name = "ecs_task_role_run_task-${random_uuid.val.id}" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ecs:DescribeTasks", + "ecs:RunTask", + "ecs:ListTasks", + "iam:PassRole", + "sqs:receivemessage", + "sqs:deletemessage", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + +data "aws_iam_policy_document" "ecs_task_role_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +// ECS Task Execution IAM role +resource "aws_iam_role" "execution_role_for_ecs_task" { + name = "execution_role_for_ecs_task-${random_uuid.val.id}" + assume_role_policy = data.aws_iam_policy_document.ecs_execution_role_assume_role.json + managed_policy_arns = [aws_iam_policy.ecs_execution_role_policy.arn] +} + +resource "aws_iam_policy" "ecs_execution_role_policy" { + name = "ecs_task_execution_role_policy-${random_uuid.val.id}" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:CreateLogGroup" + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + +data "aws_iam_policy_document" "ecs_execution_role_assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} diff --git a/terraform/compute-node-service/lambda.tf b/terraform/compute-node-service/lambda.tf index cc76c6a..fa14a69 100644 --- a/terraform/compute-node-service/lambda.tf +++ b/terraform/compute-node-service/lambda.tf @@ -17,8 +17,8 @@ resource "aws_lambda_function" "compute_node_service" { variables = { REGION = "us-east-1" CLUSTER_NAME = aws_ecs_cluster.pipeline_cluster.name - # TASK_DEFINITION_NAME = aws_ecs_task_definition.pipeline.family - # CONTAINER_NAME = aws_ecs_task_definition.pipeline.family # currently same as name of task definition + TASK_DEFINITION_NAME = aws_ecs_task_definition.pipeline.family + CONTAINER_NAME = aws_ecs_task_definition.pipeline.family # currently same as name of task definition # SUBNET_IDS = local.subnet_ids # SECURITY_GROUP_ID = aws_default_security_group.default.id } diff --git a/terraform/compute-node-service/outputs.tf b/terraform/compute-node-service/outputs.tf index d0b8783..7db918e 100644 --- a/terraform/compute-node-service/outputs.tf +++ b/terraform/compute-node-service/outputs.tf @@ -2,4 +2,10 @@ output "lambda_function_url" { description = "Lambda URL" value = aws_lambda_function_url.compute_node_service.function_url +} + +output "deploy_app_ecr_repository" { + description = "Deploy App ECR repository" + + value = aws_ecr_repository.app.repository_url } \ No newline at end of file From 5071b3460c633ca11d96b3f8d1467f2800104d9b Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 13:17:15 -0500 Subject: [PATCH 05/10] updates env vars --- terraform/compute-node-service/data.tf | 35 +++++++++++++++++++ terraform/compute-node-service/lambda.tf | 4 +-- terraform/compute-node-service/variables.tf | 4 +++ terraform/compute-node-service/vpc.tf | 38 +++++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 terraform/compute-node-service/data.tf create mode 100644 terraform/compute-node-service/variables.tf create mode 100644 terraform/compute-node-service/vpc.tf diff --git a/terraform/compute-node-service/data.tf b/terraform/compute-node-service/data.tf new file mode 100644 index 0000000..9c57767 --- /dev/null +++ b/terraform/compute-node-service/data.tf @@ -0,0 +1,35 @@ +# // policy document - gateway lambda +# data "aws_iam_policy_document" "iam_policy_document_compute" { +# statement { +# sid = "CloudwatchPermissions" +# effect = "Allow" +# actions = [ +# "logs:CreateLogGroup", +# "logs:CreateLogStream", +# "logs:PutLogEvents" +# ] +# resources = ["*"] +# } + +# statement { +# sid = "ECSTaskPermissions" +# effect = "Allow" +# actions = [ +# "ecs:DescribeTasks", +# "ecs:RunTask", +# "ecs:ListTasks" +# ] +# resources = ["*"] +# } + +# statement { +# sid = "ECSPassRole" +# effect = "Allow" +# actions = [ +# "iam:PassRole", +# ] +# resources = [ +# "*" +# ] +# } +# } \ No newline at end of file diff --git a/terraform/compute-node-service/lambda.tf b/terraform/compute-node-service/lambda.tf index fa14a69..ec0281e 100644 --- a/terraform/compute-node-service/lambda.tf +++ b/terraform/compute-node-service/lambda.tf @@ -19,8 +19,8 @@ resource "aws_lambda_function" "compute_node_service" { CLUSTER_NAME = aws_ecs_cluster.pipeline_cluster.name TASK_DEFINITION_NAME = aws_ecs_task_definition.pipeline.family CONTAINER_NAME = aws_ecs_task_definition.pipeline.family # currently same as name of task definition - # SUBNET_IDS = local.subnet_ids - # SECURITY_GROUP_ID = aws_default_security_group.default.id + SUBNET_IDS = local.subnet_ids + SECURITY_GROUP_ID = aws_default_security_group.default.id } } } diff --git a/terraform/compute-node-service/variables.tf b/terraform/compute-node-service/variables.tf new file mode 100644 index 0000000..7c6217d --- /dev/null +++ b/terraform/compute-node-service/variables.tf @@ -0,0 +1,4 @@ +variable "az" { + type = list + default = ["a", "b", "c", "d", "e", "f"] +} \ No newline at end of file diff --git a/terraform/compute-node-service/vpc.tf b/terraform/compute-node-service/vpc.tf new file mode 100644 index 0000000..078f6ee --- /dev/null +++ b/terraform/compute-node-service/vpc.tf @@ -0,0 +1,38 @@ +// Default VPC and subnet(s) +resource "aws_default_vpc" "default" { + tags = { + Name = "Default VPC" + } +} + +// TODO - create new security group +resource "aws_default_security_group" "default" { + vpc_id = aws_default_vpc.default.id + + ingress { + protocol = -1 + self = true + from_port = 0 + to_port = 0 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} +resource "aws_default_subnet" "default_az1" { + availability_zone = "us-east-1${var.az[count.index]}" + + tags = { + Name = "Default subnet for us-east-1${var.az[count.index]}" + } + count = 6 +} + +locals { + subnet_ids = join(",", aws_default_subnet.default_az1[*].id) + subnet_ids_list = aws_default_subnet.default_az1[*].id +} \ No newline at end of file From 0c893e3cbfc19b4fc7cb5763934f29334245cac9 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 13:44:30 -0500 Subject: [PATCH 06/10] invokes ECS task --- .../compute-node-service.py | 95 +++++-------------- terraform/compute-node-service/data.tf | 66 ++++++------- terraform/compute-node-service/lambda.tf | 14 --- 3 files changed, 57 insertions(+), 118 deletions(-) diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py index ebe0c56..aefa29f 100644 --- a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -3,21 +3,31 @@ import base64 import os -def lambda_handler(event, context): +ecs_client = boto3_client("ecs", region_name=os.environ['REGION']) - print(event) +def lambda_handler(event, context): + cluster_name = os.environ['CLUSTER_NAME'] + task_definition_name = os.environ['TASK_DEFINITION_NAME'] + container_name = os.environ['CONTAINER_NAME'] + security_group = os.environ['SECURITY_GROUP_ID'] + subnet_ids = os.environ['SUBNET_IDS'] + + print(event) + + if event['isBase64Encoded'] == True: + body = base64.b64decode(event['body']).decode('utf-8') + event['body'] = body + event['isBase64Encoded'] = False + + json_body = json.loads(event['body']) + print(json_body) - if event['isBase64Encoded'] == True: - body = base64.b64decode(event['body']).decode('utf-8') - event['body'] = body - event['isBase64Encoded'] = False - json_body = json.loads(event['body']) - print(json_body) + account_id = json_body['accountId'] # start Fargate task - if cluster_name != "": - print("Starting Fargate task") - response = ecs_client.run_task( + if cluster_name != "": + print("Starting Fargate task") + response = ecs_client.run_task( cluster = cluster_name, launchType = 'FARGATE', taskDefinition=task_definition_name, @@ -35,73 +45,16 @@ def lambda_handler(event, context): { 'name': container_name, 'environment': [ - { - 'name': 'INTEGRATION_ID', - 'value': integration_id - }, - { - 'name': 'BASE_DIR', - 'value': '/mnt/efs' - }, - { - 'name': 'TASK_DEFINITION_NAME_POST', - 'value': task_definition_name_post - }, - { - 'name': 'CONTAINER_NAME_POST', - 'value': container_name_post - }, - { - 'name': 'PENNSIEVE_API_KEY', - 'value': api_key - }, - { - 'name': 'PENNSIEVE_API_SECRET', - 'value': api_secret - }, - { - 'name': 'PENNSIEVE_API_HOST', - 'value': pennsieve_host - }, - { - 'name': 'PENNSIEVE_API_HOST2', - 'value': pennsieve_host2 - }, - { - 'name': 'PENNSIEVE_AGENT_HOME', - 'value': pennieve_agent_home - }, - { - 'name': 'PENNSIEVE_UPLOAD_BUCKET', - 'value': pennsieve_upload_bucket - }, - { - 'name': 'CLUSTER_NAME', - 'value': cluster_name - }, - { - 'name': 'SECURITY_GROUP_ID', - 'value': security_group - }, - { - 'name': 'SUBNET_IDS', - 'value': subnet_ids - }, - { - 'name': 'SESSION_TOKEN', - 'value': session_token - }, { - 'name': 'ENVIRONMENT', - 'value': environment + 'name': 'ACCOUNT_ID', + 'value': account_id }, - ], }, ], }) - return { + return { 'statusCode': 202, 'body': json.dumps(str('Compute node creation initiated')) } \ No newline at end of file diff --git a/terraform/compute-node-service/data.tf b/terraform/compute-node-service/data.tf index 9c57767..21a4133 100644 --- a/terraform/compute-node-service/data.tf +++ b/terraform/compute-node-service/data.tf @@ -1,35 +1,35 @@ -# // policy document - gateway lambda -# data "aws_iam_policy_document" "iam_policy_document_compute" { -# statement { -# sid = "CloudwatchPermissions" -# effect = "Allow" -# actions = [ -# "logs:CreateLogGroup", -# "logs:CreateLogStream", -# "logs:PutLogEvents" -# ] -# resources = ["*"] -# } +// policy document - compute lambda +data "aws_iam_policy_document" "iam_policy_document_compute" { + statement { + sid = "CloudwatchPermissions" + effect = "Allow" + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = ["*"] + } -# statement { -# sid = "ECSTaskPermissions" -# effect = "Allow" -# actions = [ -# "ecs:DescribeTasks", -# "ecs:RunTask", -# "ecs:ListTasks" -# ] -# resources = ["*"] -# } + statement { + sid = "ECSTaskPermissions" + effect = "Allow" + actions = [ + "ecs:DescribeTasks", + "ecs:RunTask", + "ecs:ListTasks" + ] + resources = ["*"] + } -# statement { -# sid = "ECSPassRole" -# effect = "Allow" -# actions = [ -# "iam:PassRole", -# ] -# resources = [ -# "*" -# ] -# } -# } \ No newline at end of file + statement { + sid = "ECSPassRole" + effect = "Allow" + actions = [ + "iam:PassRole", + ] + resources = [ + "*" + ] + } +} \ No newline at end of file diff --git a/terraform/compute-node-service/lambda.tf b/terraform/compute-node-service/lambda.tf index ec0281e..fdf2334 100644 --- a/terraform/compute-node-service/lambda.tf +++ b/terraform/compute-node-service/lambda.tf @@ -87,20 +87,6 @@ resource "aws_s3_object" "compute_node_service_lambda" { etag = filemd5(data.archive_file.compute_node_service_lambda.output_path) } -// policy document - compute service lambda -data "aws_iam_policy_document" "iam_policy_document_compute" { - statement { - sid = "CloudwatchPermissions" - effect = "Allow" - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = ["*"] - } -} - // S3 bucket resource "aws_s3_bucket" "lambda_bucket" { bucket = random_uuid.val.id From 7b32c9e34ab806c4bd73eddb2092595f453f9d7d Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 15:49:42 -0500 Subject: [PATCH 07/10] creates user and inline policy --- .../compute-node-service.py | 80 ++++++++++++++++++- terraform/compute-node-service/data.tf | 14 ++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py index aefa29f..bfcabef 100644 --- a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -1,9 +1,13 @@ from boto3 import client as boto3_client +import boto3 import json import base64 import os +import time +from botocore.exceptions import ClientError ecs_client = boto3_client("ecs", region_name=os.environ['REGION']) +iam_resource = boto3.resource("iam") def lambda_handler(event, context): cluster_name = os.environ['CLUSTER_NAME'] @@ -24,6 +28,73 @@ def lambda_handler(event, context): account_id = json_body['accountId'] + # create user and role + user = None + username = f"deploy-user-{account_id}" + if user_exists(username): + print("user already exists") + else: + try: + try: + user = iam_resource.create_user(UserName=username) + print(f"Created user {user.name}.") + except ClientError as error: + print( + f"Couldn't create a user. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + + try: + user_key = user.create_access_key_pair() + print(f"Created access key pair for user.") + except ClientError as error: + print( + f"Couldn't create access keys for user {user.name}. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + + time.sleep(10) # wait for user to be created + + try: + sts_client = boto3_client( + "sts", aws_access_key_id=user_key.id, aws_secret_access_key=user_key.secret + ) + deploy_account_id = sts_client.get_caller_identity()["Account"] + print(f"deploy_account_id: {deploy_account_id}") + role_arn = f"arn:aws:iam::{account_id}:role/ROLE-{deploy_account_id}" + + user.create_policy( + PolicyName=f"deploy-user-policy-{account_id}", + PolicyDocument=json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": role_arn, + } + ], + } + ), + ) + print( + f"Created an inline policy for {user.name} that lets the user assume " + f"the role." + ) + except ClientError as error: + print( + f"Couldn't create an inline policy for user {user.name}. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + + time.sleep(10) # wait + except Exception: + print("something went terribly wrong!") + # start Fargate task if cluster_name != "": print("Starting Fargate task") @@ -57,4 +128,11 @@ def lambda_handler(event, context): return { 'statusCode': 202, 'body': json.dumps(str('Compute node creation initiated')) - } \ No newline at end of file + } + +def user_exists(user_name): + try: + iam_resource.get_user(UserName=user_name) + return True + except Exception: + return False \ No newline at end of file diff --git a/terraform/compute-node-service/data.tf b/terraform/compute-node-service/data.tf index 21a4133..b0c6e69 100644 --- a/terraform/compute-node-service/data.tf +++ b/terraform/compute-node-service/data.tf @@ -32,4 +32,18 @@ data "aws_iam_policy_document" "iam_policy_document_compute" { "*" ] } + + statement { + sid = "IAMPermissions" + effect = "Allow" + actions = [ + "iam:CreateUser", + "iam:GetUser", + "iam:CreateAccessKey", + "iam:PutUserPolicy" + ] + resources = [ + "*" + ] + } } \ No newline at end of file From c390ea0214c5b0df7746771b613c9d367db4cf18 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 26 Feb 2024 18:25:24 -0500 Subject: [PATCH 08/10] assumes role --- .../compute-node-service.py | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py index bfcabef..50e8038 100644 --- a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -57,13 +57,14 @@ def lambda_handler(event, context): time.sleep(10) # wait for user to be created - try: - sts_client = boto3_client( + sts_client = boto3_client( "sts", aws_access_key_id=user_key.id, aws_secret_access_key=user_key.secret ) + + try: deploy_account_id = sts_client.get_caller_identity()["Account"] print(f"deploy_account_id: {deploy_account_id}") - role_arn = f"arn:aws:iam::{account_id}:role/ROLE-{deploy_account_id}" + assume_role_arn = f"arn:aws:iam::{account_id}:role/ROLE-{deploy_account_id}" user.create_policy( PolicyName=f"deploy-user-policy-{account_id}", @@ -74,7 +75,7 @@ def lambda_handler(event, context): { "Effect": "Allow", "Action": "sts:AssumeRole", - "Resource": role_arn, + "Resource": assume_role_arn, } ], } @@ -92,8 +93,46 @@ def lambda_handler(event, context): raise time.sleep(10) # wait + + # Assume role + try: + print("assuming role") + new_sts_client = boto3_client( + "sts", aws_access_key_id=user_key.id, aws_secret_access_key=user_key.secret + ) + response = new_sts_client.assume_role( + RoleArn=assume_role_arn, RoleSessionName=f"AssumeRole{account_id}" + ) + temp_credentials = response["Credentials"] + print(f"Assumed role {assume_role_arn} and got temporary credentials.") + except ClientError as error: + print( + f"Couldn't assume role {assume_role_arn}. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + + # list buckets + try: + print("listing buckets") + s3_resource = boto3.resource( + "s3", + aws_access_key_id=temp_credentials["AccessKeyId"], + aws_secret_access_key=temp_credentials["SecretAccessKey"], + aws_session_token=temp_credentials["SessionToken"], + ) + for bucket in s3_resource.buckets.all(): + print(bucket.name) + except ClientError as error: + print( + f"Couldn't list buckets for the account. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + except Exception: - print("something went terribly wrong!") + print("something went terribly wrong!") + # start Fargate task if cluster_name != "": @@ -129,7 +168,8 @@ def lambda_handler(event, context): 'statusCode': 202, 'body': json.dumps(str('Compute node creation initiated')) } - + +# TODO: fix error handling def user_exists(user_name): try: iam_resource.get_user(UserName=user_name) From ecf069dc6765040ef68dc6d6d3af488dd1552b55 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Tue, 27 Feb 2024 21:53:07 -0500 Subject: [PATCH 09/10] assumes role --- .../compute-node-service.py | 73 ++++++++----------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py index 50e8038..05228e9 100644 --- a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -31,6 +31,8 @@ def lambda_handler(event, context): # create user and role user = None username = f"deploy-user-{account_id}" + user_key = None + if user_exists(username): print("user already exists") else: @@ -93,47 +95,28 @@ def lambda_handler(event, context): raise time.sleep(10) # wait - - # Assume role - try: - print("assuming role") - new_sts_client = boto3_client( - "sts", aws_access_key_id=user_key.id, aws_secret_access_key=user_key.secret - ) - response = new_sts_client.assume_role( - RoleArn=assume_role_arn, RoleSessionName=f"AssumeRole{account_id}" - ) - temp_credentials = response["Credentials"] - print(f"Assumed role {assume_role_arn} and got temporary credentials.") - except ClientError as error: - print( - f"Couldn't assume role {assume_role_arn}. Here's why: " - f"{error.response['Error']['Message']}" - ) - raise - - # list buckets - try: - print("listing buckets") - s3_resource = boto3.resource( - "s3", - aws_access_key_id=temp_credentials["AccessKeyId"], - aws_secret_access_key=temp_credentials["SecretAccessKey"], - aws_session_token=temp_credentials["SessionToken"], - ) - for bucket in s3_resource.buckets.all(): - print(bucket.name) - except ClientError as error: - print( - f"Couldn't list buckets for the account. Here's why: " - f"{error.response['Error']['Message']}" - ) - raise - except Exception: print("something went terribly wrong!") - + temp_credentials = None + # Assume role + try: + print("assuming role") + new_sts_client = boto3_client( + "sts", aws_access_key_id=user_key.id, aws_secret_access_key=user_key.secret + ) + response = new_sts_client.assume_role( + RoleArn=assume_role_arn, RoleSessionName=f"AssumeRole{account_id}" + ) + temp_credentials = response["Credentials"] + print(f"Assumed role {assume_role_arn} and got temporary credentials.") + except ClientError as error: + print( + f"Couldn't assume role {assume_role_arn}. Here's why: " + f"{error.response['Error']['Message']}" + ) + raise + # start Fargate task if cluster_name != "": print("Starting Fargate task") @@ -156,9 +139,17 @@ def lambda_handler(event, context): 'name': container_name, 'environment': [ { - 'name': 'ACCOUNT_ID', - 'value': account_id - }, + 'name': 'AWS_ACCESS_KEY_ID', + 'value': temp_credentials["AccessKeyId"] + }, + { + 'name': 'AWS_SECRET_ACCESS_KEY', + 'value': temp_credentials["SecretAccessKey"] + }, + { + 'name': 'AWS_SESSION_TOKEN', + 'value': temp_credentials["SessionToken"] + }, ], }, ], From 4f61336b3650ae2e7f610f7b73d69e72b993e0e2 Mon Sep 17 00:00:00 2001 From: Edmore Moyo Date: Mon, 4 Mar 2024 21:41:38 -0500 Subject: [PATCH 10/10] application deployment params --- .../compute-node-service.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py index 05228e9..eead9d3 100644 --- a/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py +++ b/terraform/compute-node-service/compute-node-service-lambda/compute-node-service.py @@ -27,6 +27,8 @@ def lambda_handler(event, context): print(json_body) account_id = json_body['accountId'] + source_url = json_body['sourceUrl'] + destination = json_body['destination'] # create user and role user = None @@ -116,6 +118,24 @@ def lambda_handler(event, context): f"{error.response['Error']['Message']}" ) raise + + # list buckets + # try: + # print("listing buckets") + # s3_resource = boto3.resource( + # "s3", + # aws_access_key_id=temp_credentials["AccessKeyId"], + # aws_secret_access_key=temp_credentials["SecretAccessKey"], + # aws_session_token=temp_credentials["SessionToken"], + # ) + # for bucket in s3_resource.buckets.all(): + # print(bucket.name) + # except ClientError as error: + # print( + # f"Couldn't list buckets for the account. Here's why: " + # f"{error.response['Error']['Message']}" + # ) + # raise # start Fargate task if cluster_name != "": @@ -137,6 +157,7 @@ def lambda_handler(event, context): 'containerOverrides': [ { 'name': container_name, + 'command': ['--context', source_url, "--destination", destination, "--force"], 'environment': [ { 'name': 'AWS_ACCESS_KEY_ID', @@ -166,4 +187,4 @@ def user_exists(user_name): iam_resource.get_user(UserName=user_name) return True except Exception: - return False \ No newline at end of file + return False \ No newline at end of file