diff --git a/charts/movement-node/templates/ingress.yaml b/charts/movement-node/templates/ingress.yaml new file mode 100644 index 0000000..ac1e988 --- /dev/null +++ b/charts/movement-node/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled }} +{{- if or (eq .Values.node.type "vfn") (eq .Values.node.type "fullnode") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "movement-node.fullname" . }} + labels: + {{- include "movement-node.labels" . | nindent 4 }} + annotations: + {{- if .Values.ingress.annotations }} + {{- toYaml .Values.ingress.annotations | nindent 4 }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className | default "nginx" }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.hostname }} + {{- if .Values.ingress.tls.secretName }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- else if .Values.ingress.tls.wildcardSecretName }} + secretName: {{ .Values.ingress.tls.wildcardSecretName }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.ingress.hostname }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "movement-node.fullname" . }} + port: + number: {{ .Values.service.ports.api }} +{{- end }} +{{- end }} diff --git a/charts/movement-node/templates/service.yaml b/charts/movement-node/templates/service.yaml index 5bf27da..972c963 100644 --- a/charts/movement-node/templates/service.yaml +++ b/charts/movement-node/templates/service.yaml @@ -7,7 +7,7 @@ metadata: {{- if .Values.service.annotations }} annotations: {{- toYaml .Values.service.annotations | nindent 4 }} -{{- else if and (or (eq .Values.node.type "vfn") (eq .Values.node.type "fullnode")) .Values.loadBalancer.annotations }} +{{- else if and (or (eq .Values.node.type "vfn") (eq .Values.node.type "fullnode")) (not .Values.ingress.enabled) .Values.loadBalancer.annotations }} annotations: {{- toYaml .Values.loadBalancer.annotations | nindent 4 }} {{- end }} @@ -16,6 +16,8 @@ spec: type: {{ .Values.service.type }} {{- else if eq .Values.node.type "validator" }} type: ClusterIP +{{- else if .Values.ingress.enabled }} + type: ClusterIP {{- else }} type: LoadBalancer {{- end }} diff --git a/charts/movement-node/values.yaml b/charts/movement-node/values.yaml index 4535ef8..499632e 100644 --- a/charts/movement-node/values.yaml +++ b/charts/movement-node/values.yaml @@ -104,12 +104,31 @@ service: # Load balancer configuration (for vfn and fullnode) loadBalancer: - # Auto-enabled for vfn and fullnode + # Auto-enabled for vfn and fullnode (when ingress is disabled) enabled: false # Leave false for auto-config annotations: service.beta.kubernetes.io/aws-load-balancer-type: "nlb" service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" +# Ingress configuration (for HTTPS access to API) +ingress: + # Enable ingress for TLS termination (recommended for production) + enabled: false + # Ingress class name (nginx, traefik, etc.) + className: "nginx" + # Hostname for this node (e.g., node-01.nodeinfra.testnet.movementnetwork.xyz) + hostname: "" + # Additional ingress annotations + annotations: {} + # TLS configuration + tls: + enabled: true + # Use a shared wildcard certificate secret (recommended) + # This secret is created by cert-manager and synced by reflector + wildcardSecretName: "wildcard-tls" + # Or use a per-node certificate secret + secretName: "" + # Monitoring configuration monitoring: enabled: true diff --git a/examples/public-fullnode/.env.example b/examples/public-fullnode/.env.example index 3a01924..8e6dd1b 100644 --- a/examples/public-fullnode/.env.example +++ b/examples/public-fullnode/.env.example @@ -30,3 +30,23 @@ BOOTSTRAP_S3_REGION=us-west-2 # FULLNODE_NAMESPACE=movement-l1 # FULLNODE_SERVICE_NAME=public-fullnode # FULLNODE_CONFIG_FILE=charts/movement-node/files/fullnode.yaml + +# Ingress Configuration (TLS) +# Enable HTTPS ingress instead of direct LoadBalancer exposure. +# When enabled: +# - Terraform provisions: NGINX Ingress Controller, cert-manager, wildcard TLS certificate +# - cert-manager uses Route53 DNS-01 challenge +# - Fullnode service uses ClusterIP +# - Traffic flows: HTTPS:443 → NGINX Ingress (TLS termination) → Service:8080 +# +# CHAIN_NAME: Network name subdomain (default: testnet) +# INGRESS_DOMAIN: Base domain / Route53 zone (default: scratchpad.movementnetwork.xyz) +# +# INGRESS_ENABLED=true +# CHAIN_NAME=testnet +# INGRESS_DOMAIN=scratchpad.movementnetwork.xyz +# +# With INGRESS_ENABLED=true, CHAIN_NAME=testnet, and FULLNODE_SERVICE_NAME=public-fullnode: +# → Fullnode accessible at https://public-fullnode.testnet.scratchpad.movementnetwork.xyz +# +INGRESS_ENABLED=false diff --git a/examples/public-fullnode/deploy.py b/examples/public-fullnode/deploy.py index 1aa28ac..b7da7a1 100755 --- a/examples/public-fullnode/deploy.py +++ b/examples/public-fullnode/deploy.py @@ -59,6 +59,13 @@ def build_terraform_vars(env_vars: dict) -> dict: t.strip() for t in env_vars["NODE_INSTANCE_TYPES"].split(",") ] + # Ingress configuration + enable_ingress = env_vars.get("INGRESS_ENABLED", "false").lower() in ("true", "1", "yes") + variables["enable_ingress"] = enable_ingress + if enable_ingress: + variables["chain_name"] = env_vars.get("CHAIN_NAME", "testnet") + variables["ingress_domain"] = env_vars.get("INGRESS_DOMAIN", "scratchpad.movementnetwork.xyz") + return variables @@ -101,6 +108,26 @@ def build_helm_config(env_vars: dict, outputs: dict) -> dict: "storage.parameters.throughput": "500", } + # Ingress configuration for TLS (check outputs first, then env vars) + ingress_enabled = outputs.get("ingress_enabled", False) + if not ingress_enabled: + ingress_enabled = env_vars.get("INGRESS_ENABLED", "false").lower() in ("true", "1", "yes") + chain_name = env_vars.get("CHAIN_NAME", "testnet") + ingress_base_domain = env_vars.get("INGRESS_DOMAIN", "scratchpad.movementnetwork.xyz") + ingress_domain = f"{chain_name}.{ingress_base_domain}" + + if ingress_enabled: + ingress_hostname = f"{service_name}.{ingress_domain}" + set_values.update({ + "ingress.enabled": "true", + "ingress.hostname": ingress_hostname, + "ingress.className": "nginx", + "ingress.tls.enabled": "true", + # TLS secret uses chart default (wildcard-tls) unless INGRESS_TLS_SECRET is set + }) + if env_vars.get("INGRESS_TLS_SECRET"): + set_values["ingress.tls.wildcardSecretName"] = env_vars["INGRESS_TLS_SECRET"] + # Add bootstrap if enabled if outputs.get("fullnode_bootstrap_enabled"): # Parse S3 URI to extract bucket and prefix @@ -145,12 +172,18 @@ def deploy(env_vars: dict, force_create: bool, validate: bool) -> None: outputs = cluster.terraform.get_outputs() or {} helm_config = build_helm_config(env_vars, outputs) + # Check if ingress is requested + ingress_enabled = outputs.get("ingress_enabled", False) + if not ingress_enabled: + ingress_enabled = env_vars.get("INGRESS_ENABLED", "false").lower() in ("true", "1", "yes") + cluster.deploy( env_vars=env_vars, terraform_vars=terraform_vars, helm_config=helm_config, skip_if_exists=not force_create, validate=validate, + ingress_enabled=ingress_enabled, ) diff --git a/examples/public-fullnode/main.tf b/examples/public-fullnode/main.tf index 070a31f..a1645e7 100644 --- a/examples/public-fullnode/main.tf +++ b/examples/public-fullnode/main.tf @@ -3,6 +3,66 @@ provider "aws" { region = var.region } +# Kubernetes provider (configured after EKS is created) +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_ca_certificate) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_id, + "--region", + var.region + ] + } +} + +# Helm provider +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_ca_certificate) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_id, + "--region", + var.region + ] + } + } +} + +# kubectl provider +provider "kubectl" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_ca_certificate) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_id, + "--region", + var.region + ] + } +} + locals { fullnode_bootstrap_enabled = var.fullnode_bootstrap_s3_bucket != "" fullnode_bootstrap_prefix = trim(var.fullnode_bootstrap_s3_prefix, "/") @@ -19,8 +79,8 @@ module "network" { validator_name = var.validator_name region = var.region vpc_cidr = var.vpc_cidr - dns_enabled = var.enable_dns - dns_zone_name = var.dns_zone_name + dns_enabled = var.enable_dns || var.enable_ingress + dns_zone_name = var.enable_dns ? var.dns_zone_name : (var.enable_ingress ? var.ingress_domain : "") # Cost optimization: single NAT gateway for demo single_nat_gateway = true @@ -107,3 +167,21 @@ resource "aws_iam_role_policy" "fullnode_s3_read" { role = aws_iam_role.fullnode_s3[0].id policy = data.aws_iam_policy_document.fullnode_s3_read[0].json } + +# Ingress infrastructure (NGINX Ingress Controller + cert-manager + wildcard TLS) +module "ingress" { + source = "../../terraform-modules/movement-ingress" + count = var.enable_ingress ? 1 : 0 + + cluster_name = module.eks.cluster_name + cluster_oidc_issuer_url = "https://${module.eks.oidc_provider_url}" + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + route53_zone_id = module.network.dns_zone_id + route53_zone_name = var.ingress_domain + wildcard_domain = "*.${var.chain_name}.${var.ingress_domain}" + node_namespace = var.fullnode_namespace + + tags = var.tags + + depends_on = [module.eks] +} diff --git a/examples/public-fullnode/outputs.tf b/examples/public-fullnode/outputs.tf index 689deae..5acf0b3 100644 --- a/examples/public-fullnode/outputs.tf +++ b/examples/public-fullnode/outputs.tf @@ -57,3 +57,19 @@ output "configure_kubectl" { description = "Command to configure kubectl" value = "aws eks update-kubeconfig --region ${var.region} --name ${module.eks.cluster_name}" } + +# Ingress outputs (when enabled) +output "ingress_enabled" { + description = "Whether ingress is enabled" + value = var.enable_ingress +} + +output "ingress_base_domain" { + description = "Base domain for ingress (Route53 zone, not including chain_name prefix)" + value = var.enable_ingress ? var.ingress_domain : "" +} + +output "ingress_namespace" { + description = "NGINX Ingress Controller namespace" + value = var.enable_ingress ? module.ingress[0].ingress_namespace : "" +} diff --git a/examples/public-fullnode/variables.tf b/examples/public-fullnode/variables.tf index 4f9cc43..9e698b5 100644 --- a/examples/public-fullnode/variables.tf +++ b/examples/public-fullnode/variables.tf @@ -91,3 +91,22 @@ variable "fullnode_dns_name" { type = string default = "" } + +# Ingress Configuration +variable "enable_ingress" { + description = "Enable NGINX Ingress Controller with TLS for fullnode access" + type = bool + default = false +} + +variable "chain_name" { + description = "Chain/network name for ingress subdomain (e.g., testnet, mainnet)" + type = string + default = "testnet" +} + +variable "ingress_domain" { + description = "Base domain for ingress (Route53 zone)" + type = string + default = "scratchpad.movementnetwork.xyz" +} diff --git a/examples/public-fullnode/versions.tf b/examples/public-fullnode/versions.tf index 831bef0..aa26df2 100644 --- a/examples/public-fullnode/versions.tf +++ b/examples/public-fullnode/versions.tf @@ -6,5 +6,17 @@ terraform { source = "hashicorp/aws" version = ">= 5.0, < 7.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.35" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } diff --git a/examples/validator-vfn/.env.example b/examples/validator-vfn/.env.example index 8de3f98..c8626ca 100644 --- a/examples/validator-vfn/.env.example +++ b/examples/validator-vfn/.env.example @@ -77,3 +77,23 @@ DEPLOY_FULLNODE=false VALIDATOR_KEYS_SECRET_NAME= # This is K8s secrete name where used by k8s for indexing the secrets VALIDATOR_KEYS_SECRET=validator-identity + +# Ingress Configuration (TLS) +# Enable HTTPS ingress instead of direct LoadBalancer exposure. +# When enabled: +# - Terraform provisions: NGINX Ingress Controller, cert-manager, wildcard TLS certificate +# - cert-manager uses Route53 DNS-01 challenge +# - VFN/fullnode services use ClusterIP +# - Traffic flows: HTTPS:443 → NGINX Ingress (TLS termination) → Service:8080 +# +# CHAIN_NAME: Network name subdomain (default: testnet) +# INGRESS_DOMAIN: Base domain / Route53 zone (default: scratchpad.movementnetwork.xyz) +# +# INGRESS_ENABLED=true +# CHAIN_NAME=testnet +# INGRESS_DOMAIN=scratchpad.movementnetwork.xyz +# +# With INGRESS_ENABLED=true, CHAIN_NAME=testnet, and VFN_NAME=vfn-01: +# → VFN accessible at https://vfn-01.testnet.scratchpad.movementnetwork.xyz +# +INGRESS_ENABLED=false diff --git a/examples/validator-vfn/deploy.py b/examples/validator-vfn/deploy.py index 85be94e..e20893f 100644 --- a/examples/validator-vfn/deploy.py +++ b/examples/validator-vfn/deploy.py @@ -197,6 +197,13 @@ def build_terraform_vars(env_vars: dict) -> dict: variables["node_max_size"] = node_count + 2 variables["tags"] = {"Validator": validator_name} + # Ingress configuration + enable_ingress = env_vars.get("INGRESS_ENABLED", "false").lower() in ("true", "1", "yes") + variables["enable_ingress"] = enable_ingress + if enable_ingress: + variables["chain_name"] = env_vars.get("CHAIN_NAME", "testnet") + variables["ingress_domain"] = env_vars.get("INGRESS_DOMAIN", "scratchpad.movementnetwork.xyz") + return variables @@ -254,6 +261,8 @@ def deploy_node( vfn_service: str | None = None, validator_keys_secret: str | None = None, vfn_keys_secret: str | None = None, + ingress_enabled: bool = False, + ingress_hostname: str | None = None, ) -> None: """Deploy a single node with appropriate configuration.""" info(f"Deploying {node_type}: {node_name}") @@ -328,6 +337,17 @@ def deploy_node( set_values["service.type"] = service_type info(f" Service type: {service_type}") + # Ingress configuration for TLS termination + if ingress_enabled and ingress_hostname: + set_values["ingress.enabled"] = "true" + set_values["ingress.hostname"] = ingress_hostname + set_values["ingress.className"] = "nginx" + set_values["ingress.tls.enabled"] = "true" + # Only override TLS secret if explicitly set; otherwise use chart default (wildcard-tls) + if env_vars.get("INGRESS_TLS_SECRET"): + set_values["ingress.tls.wildcardSecretName"] = env_vars["INGRESS_TLS_SECRET"] + info(f" Ingress: https://{ingress_hostname}") + # Node-specific configuration if node_type == "validator": if not validator_keys_secret: @@ -402,6 +422,12 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa deploy_vfn = env_vars.get("DEPLOY_VFN", "true").lower() in ("true", "1", "yes") deploy_fullnode = env_vars.get("DEPLOY_FULLNODE", "false").lower() in ("true", "1", "yes") + # Ingress configuration for TLS + ingress_enabled = env_vars.get("INGRESS_ENABLED", "false").lower() in ("true", "1", "yes") + chain_name = env_vars.get("CHAIN_NAME", "testnet") + ingress_base_domain = env_vars.get("INGRESS_DOMAIN", "scratchpad.movementnetwork.xyz") + ingress_domain = f"{chain_name}.{ingress_base_domain}" + # Deployment names validator_name = env_vars.get("VALIDATOR_NAME", "validator-01") vfn_name = env_vars.get("VFN_NAME", "vfn-01") @@ -413,7 +439,16 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa # Display deployment plan info("Deployment Topology:") info(f" Validator: {validator_name} (ClusterIP - private)") - if deploy_vfn and deploy_fullnode: + if ingress_enabled: + info(f" Ingress: ENABLED (TLS via *.{ingress_domain})") + if deploy_vfn and deploy_fullnode: + info(f" VFN: {vfn_name} (ClusterIP → Ingress)") + info(f" Fullnode: {fullnode_name} (ClusterIP → Ingress)") + info(" → 3-tier setup: External clients access via HTTPS ingress") + elif deploy_vfn: + info(f" VFN: {vfn_name} (ClusterIP → Ingress)") + info(" → 2-tier setup: External clients access VFN via HTTPS ingress") + elif deploy_vfn and deploy_fullnode: info(f" VFN: {vfn_name} (ClusterIP - private)") info(f" Fullnode: {fullnode_name} (LoadBalancer - public)") info(" → 3-tier setup: External clients access fullnode") @@ -449,9 +484,17 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa terraform_vars = build_terraform_vars(env_vars) outputs = cluster.terraform.get_outputs() or {} - if not force_create and outputs: + # Check if we need to run Terraform: + # - force_create is set + # - no outputs exist (fresh deployment) + # - ingress requested but not yet provisioned + ingress_needs_provisioning = ingress_enabled and not outputs.get("ingress_enabled", False) + + if not force_create and outputs and not ingress_needs_provisioning: info("Infrastructure already exists, skipping Terraform") else: + if ingress_needs_provisioning: + info("Ingress enabled but not yet provisioned, running Terraform") cluster.terraform.init(upgrade=True) cluster.terraform.validate() var_args = cluster.terraform.build_var_args(terraform_vars) @@ -471,6 +514,10 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa eks.wait_until_active() eks.update_kubeconfig() + # Update ingress_enabled from terraform outputs (actual provisioned state) + # Note: ingress_domain is NOT updated from outputs - it uses chain_name + base domain + ingress_enabled = outputs.get("ingress_enabled", ingress_enabled) + # Step 1.5: Create identity secrets from local files info("\n" + "=" * 80) info("Creating Kubernetes Identity Secrets") @@ -520,8 +567,14 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa # Deploy VFN if requested if deploy_vfn: - # VFN service type depends on whether fullnode is deployed - vfn_service_type = "ClusterIP" if deploy_fullnode else "LoadBalancer" + # VFN service type: ClusterIP if fullnode deployed OR ingress enabled, else LoadBalancer + if ingress_enabled or deploy_fullnode: + vfn_service_type = None # Let Helm chart decide (ClusterIP when ingress enabled) + else: + vfn_service_type = "LoadBalancer" + + # Build ingress hostname for VFN + vfn_ingress_hostname = f"{vfn_name}.{ingress_domain}" if ingress_enabled and ingress_domain else None deploy_node( helm=helm, @@ -533,10 +586,18 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa service_type=vfn_service_type, validator_service=validator_name, vfn_keys_secret=vfn_keys_secret, + ingress_enabled=ingress_enabled, + ingress_hostname=vfn_ingress_hostname, ) # Deploy fullnode if requested if deploy_fullnode: + # Fullnode service type: ClusterIP if ingress enabled, else LoadBalancer + fullnode_service_type = None if ingress_enabled else "LoadBalancer" + + # Build ingress hostname for fullnode + fullnode_ingress_hostname = f"{fullnode_name}.{ingress_domain}" if ingress_enabled and ingress_domain else None + deploy_node( helm=helm, node_type="fullnode", @@ -544,8 +605,10 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa namespace=namespace, validator_name=validator_name, env_vars=env_vars, - service_type="LoadBalancer", + service_type=fullnode_service_type, vfn_service=vfn_name if deploy_vfn else None, + ingress_enabled=ingress_enabled, + ingress_hostname=fullnode_ingress_hostname, ) # Step 3: Validation (if requested) @@ -574,6 +637,7 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa # Final validation: check public endpoint if available if deploy_fullnode or deploy_vfn: validate_service = fullnode_name if deploy_fullnode else vfn_name + validate_hostname = f"{validate_service}.{ingress_domain}" if ingress_enabled else None info(f"\nValidating public endpoint: {validate_service}") validate_deployment( @@ -581,6 +645,8 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa service_name=validate_service, pod_timeout=3600, lb_retries=60, + ingress_enabled=ingress_enabled, + ingress_hostname=validate_hostname, ) success("Deployment complete!") @@ -590,7 +656,14 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa info("Access Information:") info("=" * 80) - if deploy_fullnode: + if ingress_enabled: + info("\n🔒 Public Access: HTTPS via Ingress") + if deploy_fullnode: + info(f" Fullnode: https://{fullnode_name}.{ingress_domain}") + if deploy_vfn: + info(f" VFN: https://{vfn_name}.{ingress_domain}") + info(f" kubectl get ingress -n {namespace}") + elif deploy_fullnode: info("\n🌐 Public Access: Fullnode LoadBalancer") info(f" Service: {fullnode_name}") info(f" kubectl get svc {fullnode_name} -n {namespace}") diff --git a/examples/validator-vfn/main.tf b/examples/validator-vfn/main.tf index 51e2572..ca0b0a6 100644 --- a/examples/validator-vfn/main.tf +++ b/examples/validator-vfn/main.tf @@ -16,7 +16,8 @@ module "network" { validator_name = var.validator_name region = var.region vpc_cidr = var.vpc_cidr - dns_zone_name = var.enable_dns ? var.dns_zone_name : "" + dns_enabled = var.enable_dns || var.enable_ingress + dns_zone_name = var.enable_dns || var.enable_ingress ? var.ingress_domain : "" tags = merge( var.tags, @@ -63,6 +64,47 @@ provider "kubernetes" { } } +# Configure Helm provider +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_ca_certificate) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_id, + "--region", + var.region + ] + } + } +} + +# Configure kubectl provider +provider "kubectl" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_ca_certificate) + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_id, + "--region", + var.region + ] + } +} + # Create namespace for validator nodes resource "kubernetes_namespace" "movement" { metadata { @@ -73,3 +115,21 @@ resource "kubernetes_namespace" "movement" { } } } + +# Ingress infrastructure (NGINX Ingress Controller + cert-manager + wildcard TLS) +module "ingress" { + source = "../../terraform-modules/movement-ingress" + count = var.enable_ingress ? 1 : 0 + + cluster_name = module.eks.cluster_name + cluster_oidc_issuer_url = "https://${module.eks.oidc_provider_url}" + cluster_oidc_provider_arn = module.eks.oidc_provider_arn + route53_zone_id = module.network.dns_zone_id + route53_zone_name = var.ingress_domain + wildcard_domain = "*.${var.chain_name}.${var.ingress_domain}" + node_namespace = var.namespace + + tags = var.tags + + depends_on = [module.eks, kubernetes_namespace.movement] +} diff --git a/examples/validator-vfn/outputs.tf b/examples/validator-vfn/outputs.tf index f3fa1b3..5441e1b 100644 --- a/examples/validator-vfn/outputs.tf +++ b/examples/validator-vfn/outputs.tf @@ -27,3 +27,19 @@ output "kubeconfig_command" { description = "Command to update kubeconfig" value = "aws eks update-kubeconfig --region ${var.region} --name ${module.eks.cluster_id}" } + +# Ingress outputs (when enabled) +output "ingress_enabled" { + description = "Whether ingress is enabled" + value = var.enable_ingress +} + +output "ingress_base_domain" { + description = "Base domain for ingress (Route53 zone, not including chain_name prefix)" + value = var.enable_ingress ? var.ingress_domain : "" +} + +output "ingress_namespace" { + description = "NGINX Ingress Controller namespace" + value = var.enable_ingress ? module.ingress[0].ingress_namespace : "" +} diff --git a/examples/validator-vfn/variables.tf b/examples/validator-vfn/variables.tf index 0fe4e1d..eb4ff18 100644 --- a/examples/validator-vfn/variables.tf +++ b/examples/validator-vfn/variables.tf @@ -81,3 +81,22 @@ variable "tags" { type = map(string) default = {} } + +# Ingress Configuration +variable "enable_ingress" { + description = "Enable NGINX Ingress Controller with TLS for VFN/fullnode access" + type = bool + default = false +} + +variable "chain_name" { + description = "Chain/network name for ingress subdomain (e.g., testnet, mainnet)" + type = string + default = "testnet" +} + +variable "ingress_domain" { + description = "Base domain for ingress (Route53 zone)" + type = string + default = "scratchpad.movementnetwork.xyz" +} diff --git a/examples/validator-vfn/versions.tf b/examples/validator-vfn/versions.tf index 7a1bfb9..310922f 100644 --- a/examples/validator-vfn/versions.tf +++ b/examples/validator-vfn/versions.tf @@ -10,5 +10,13 @@ terraform { source = "hashicorp/kubernetes" version = "~> 2.35" } + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } diff --git a/terraform-modules/movement-ingress/iam.tf b/terraform-modules/movement-ingress/iam.tf new file mode 100644 index 0000000..321bac2 --- /dev/null +++ b/terraform-modules/movement-ingress/iam.tf @@ -0,0 +1,75 @@ +# IAM role for cert-manager to access Route53 for DNS-01 challenges + +locals { + oidc_issuer = replace(var.cluster_oidc_issuer_url, "https://", "") +} + +# IAM policy for Route53 DNS-01 challenge +data "aws_iam_policy_document" "cert_manager_route53" { + statement { + effect = "Allow" + actions = [ + "route53:GetChange" + ] + resources = ["arn:aws:route53:::change/*"] + } + + statement { + effect = "Allow" + actions = [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets" + ] + resources = ["arn:aws:route53:::hostedzone/${var.route53_zone_id}"] + } + + statement { + effect = "Allow" + actions = [ + "route53:ListHostedZonesByName" + ] + resources = ["*"] + } +} + +resource "aws_iam_policy" "cert_manager_route53" { + name = "${var.cluster_name}-cert-manager-route53" + description = "Policy for cert-manager to manage Route53 DNS records for DNS-01 challenge" + policy = data.aws_iam_policy_document.cert_manager_route53.json + + tags = var.tags +} + +# Trust policy for IRSA +data "aws_iam_policy_document" "cert_manager_assume_role" { + statement { + effect = "Allow" + principals { + type = "Federated" + identifiers = [var.cluster_oidc_provider_arn] + } + actions = ["sts:AssumeRoleWithWebIdentity"] + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:sub" + values = ["system:serviceaccount:${var.certmanager_namespace}:cert-manager"] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "cert_manager" { + name = "${var.cluster_name}-cert-manager" + assume_role_policy = data.aws_iam_policy_document.cert_manager_assume_role.json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "cert_manager_route53" { + role = aws_iam_role.cert_manager.name + policy_arn = aws_iam_policy.cert_manager_route53.arn +} diff --git a/terraform-modules/movement-ingress/main.tf b/terraform-modules/movement-ingress/main.tf new file mode 100644 index 0000000..0a6771d --- /dev/null +++ b/terraform-modules/movement-ingress/main.tf @@ -0,0 +1,188 @@ +# Reflector for syncing TLS secrets across namespaces +resource "helm_release" "reflector" { + name = "reflector" + repository = "https://emberstack.github.io/helm-charts" + chart = "reflector" + version = "7.1.288" + namespace = "kube-system" + create_namespace = false + + set { + name = "resources.requests.cpu" + value = "50m" + } + + set { + name = "resources.requests.memory" + value = "64Mi" + } +} + +# Generate a unique but predictable name for the NLB +locals { + nlb_name = "${var.cluster_name}-ingress" +} + +# NGINX Ingress Controller +resource "helm_release" "nginx_ingress" { + name = "ingress-nginx" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" + version = "4.11.3" + namespace = var.ingress_namespace + create_namespace = true + + values = [ + yamlencode({ + controller = { + replicaCount = 2 + service = { + type = "LoadBalancer" + annotations = { + "service.beta.kubernetes.io/aws-load-balancer-type" = "nlb" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" + "service.beta.kubernetes.io/aws-load-balancer-name" = local.nlb_name + } + } + resources = { + requests = { + cpu = "100m" + memory = "128Mi" + } + limits = { + cpu = "500m" + memory = "512Mi" + } + } + config = { + "use-forwarded-headers" = "true" + "compute-full-forwarded-for" = "true" + } + } + }) + ] +} + +# cert-manager for automatic TLS certificate management +resource "helm_release" "cert_manager" { + name = "cert-manager" + repository = "https://charts.jetstack.io" + chart = "cert-manager" + version = "v1.16.2" + namespace = var.certmanager_namespace + create_namespace = true + + set { + name = "crds.enabled" + value = "true" + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = aws_iam_role.cert_manager.arn + } + + depends_on = [ + aws_iam_role_policy_attachment.cert_manager_route53 + ] +} + +# ClusterIssuer for Let's Encrypt with Route53 DNS-01 challenge +resource "kubectl_manifest" "cluster_issuer" { + yaml_body = yamlencode({ + apiVersion = "cert-manager.io/v1" + kind = "ClusterIssuer" + metadata = { + name = "letsencrypt-prod" + } + spec = { + acme = { + server = "https://acme-v02.api.letsencrypt.org/directory" + email = "infrastructure@moveindustries.xyz" + privateKeySecretRef = { + name = "letsencrypt-prod-account-key" + } + solvers = [ + { + dns01 = { + route53 = { + region = data.aws_region.current.name + hostedZoneID = var.route53_zone_id + } + } + selector = { + dnsZones = [var.route53_zone_name] + } + } + ] + } + } + }) + + depends_on = [helm_release.cert_manager] +} + +# Wildcard Certificate +resource "kubectl_manifest" "wildcard_certificate" { + yaml_body = yamlencode({ + apiVersion = "cert-manager.io/v1" + kind = "Certificate" + metadata = { + name = "wildcard-tls" + namespace = var.ingress_namespace + } + spec = { + secretName = "wildcard-tls" + issuerRef = { + name = "letsencrypt-prod" + kind = "ClusterIssuer" + } + commonName = var.wildcard_domain + dnsNames = [ + var.wildcard_domain, + trimsuffix(trimprefix(var.wildcard_domain, "*."), ".") + ] + secretTemplate = { + annotations = { + "reflector.v1.k8s.emberstack.com/reflection-allowed" = "true" + "reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces" = var.node_namespace + "reflector.v1.k8s.emberstack.com/reflection-auto-enabled" = "true" + "reflector.v1.k8s.emberstack.com/reflection-auto-namespaces" = var.node_namespace + } + } + } + }) + + depends_on = [kubectl_manifest.cluster_issuer, helm_release.reflector] +} + +data "aws_region" "current" {} + +# NOTE: NLB provisioning is asynchronous. After helm_release.nginx_ingress completes, +# AWS needs 2-5 minutes to provision the actual NLB. If the first terraform apply fails +# with "no matching ELB found", wait a few minutes and rerun. This is expected behavior. +# +# The data source lookup will fail fast if NLB doesn't exist yet, which is preferable +# to blocking with arbitrary sleep times that may still be insufficient. + +# Get the NLB by name (set via service annotation) +data "aws_lb" "nginx_ingress" { + name = local.nlb_name + + depends_on = [helm_release.nginx_ingress] +} + +# Create DNS record for the wildcard domain pointing to the NLB +# Using ALIAS record for better performance and native AWS integration +resource "aws_route53_record" "wildcard" { + zone_id = var.route53_zone_id + name = var.wildcard_domain + type = "A" + + alias { + name = data.aws_lb.nginx_ingress.dns_name + zone_id = data.aws_lb.nginx_ingress.zone_id + evaluate_target_health = true + } +} diff --git a/terraform-modules/movement-ingress/outputs.tf b/terraform-modules/movement-ingress/outputs.tf new file mode 100644 index 0000000..5a5a92b --- /dev/null +++ b/terraform-modules/movement-ingress/outputs.tf @@ -0,0 +1,39 @@ +output "ingress_namespace" { + description = "Namespace where NGINX Ingress Controller is deployed" + value = var.ingress_namespace +} + +output "cert_manager_namespace" { + description = "Namespace where cert-manager is deployed" + value = var.certmanager_namespace +} + +output "cert_manager_role_arn" { + description = "IAM role ARN for cert-manager" + value = aws_iam_role.cert_manager.arn +} + +output "wildcard_certificate_secret" { + description = "Name of the Kubernetes secret containing the wildcard TLS certificate" + value = "wildcard-tls" +} + +output "cluster_issuer_name" { + description = "Name of the ClusterIssuer for Let's Encrypt" + value = "letsencrypt-prod" +} + +output "ingress_class_name" { + description = "Ingress class name to use in Ingress resources" + value = "nginx" +} + +output "load_balancer_hostname" { + description = "NLB hostname for the NGINX Ingress Controller" + value = data.aws_lb.nginx_ingress.dns_name +} + +output "wildcard_dns_record" { + description = "DNS record name for the wildcard domain" + value = aws_route53_record.wildcard.name +} diff --git a/terraform-modules/movement-ingress/variables.tf b/terraform-modules/movement-ingress/variables.tf new file mode 100644 index 0000000..1ed6915 --- /dev/null +++ b/terraform-modules/movement-ingress/variables.tf @@ -0,0 +1,55 @@ +variable "cluster_name" { + description = "EKS cluster name" + type = string +} + +variable "cluster_oidc_issuer_url" { + description = "EKS cluster OIDC issuer URL (for IRSA)" + type = string +} + +variable "cluster_oidc_provider_arn" { + description = "EKS cluster OIDC provider ARN (for IRSA)" + type = string +} + +variable "route53_zone_id" { + description = "Route53 hosted zone ID for DNS-01 challenge" + type = string + default = "Z01726943LS9E7WG35N3V" # scratchpad.movementnetwork.xyz +} + +variable "route53_zone_name" { + description = "Route53 hosted zone name" + type = string + default = "scratchpad.movementnetwork.xyz" +} + +variable "wildcard_domain" { + description = "Wildcard domain for TLS certificate (e.g., *.{chain_name}.{ingress_domain})" + type = string +} + +variable "ingress_namespace" { + description = "Namespace for NGINX Ingress Controller" + type = string + default = "ingress-nginx" +} + +variable "certmanager_namespace" { + description = "Namespace for cert-manager" + type = string + default = "cert-manager" +} + +variable "node_namespace" { + description = "Namespace where movement nodes are deployed (for TLS secret sync)" + type = string + default = "movement-l1" +} + +variable "tags" { + description = "Common tags for all resources" + type = map(string) + default = {} +} diff --git a/terraform-modules/movement-ingress/versions.tf b/terraform-modules/movement-ingress/versions.tf new file mode 100644 index 0000000..5675101 --- /dev/null +++ b/terraform-modules/movement-ingress/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.0" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.14" + } + } +} diff --git a/tools/cluster.py b/tools/cluster.py index 9a6270d..4b3a064 100644 --- a/tools/cluster.py +++ b/tools/cluster.py @@ -44,6 +44,7 @@ def deploy( helm_config: dict[str, Any], skip_if_exists: bool = True, validate: bool = False, + ingress_enabled: bool = False, ) -> dict[str, Any]: """ Deploy infrastructure and workload. @@ -60,6 +61,7 @@ def deploy( - set_files: Dict of Helm set-file values skip_if_exists: Skip infra creation if cluster exists validate: Whether to validate deployment after completion + ingress_enabled: Whether ingress is requested for this deployment Returns: Dictionary of deployment information @@ -75,11 +77,16 @@ def deploy( ) region = outputs.get("region") or terraform_vars.get("region", "us-east-1") + # Check if ingress needs provisioning (requested but not yet deployed) + ingress_needs_provisioning = ingress_enabled and not outputs.get("ingress_enabled", False) + # Check if cluster exists eks = EKSManager(cluster_name, region) - if skip_if_exists and eks.cluster_exists(): + if skip_if_exists and eks.cluster_exists() and not ingress_needs_provisioning: success(f"Infrastructure already exists (cluster: {cluster_name}, region: {region})") else: + if ingress_needs_provisioning: + info("Ingress enabled but not yet provisioned, running Terraform") # Provision infrastructure self.terraform.init(upgrade=True) self.terraform.validate() @@ -137,6 +144,12 @@ def deploy( max_retries = int(env_vars.get("MAX_RETRIES", "60")) retry_interval = int(env_vars.get("RETRY_INTERVAL", "10")) + # Extract ingress hostname from helm config if ingress is enabled + ingress_hostname = None + if ingress_enabled: + set_values = helm_config.get("set_values", {}) + ingress_hostname = set_values.get("ingress.hostname") + validate_deployment( namespace=namespace, service_name=service_name, @@ -144,6 +157,8 @@ def deploy( lb_retries=max_retries, interval=retry_interval, validate_api=validate, + ingress_enabled=ingress_enabled, + ingress_hostname=ingress_hostname, ) success("Deployment completed successfully!") diff --git a/tools/terraform.py b/tools/terraform.py index 405f7ae..c0d8f5c 100644 --- a/tools/terraform.py +++ b/tools/terraform.py @@ -93,9 +93,41 @@ def apply( if var_args: cmd.extend(var_args) - run_command(cmd, cwd=self.working_dir) + # Run with check=False to handle errors with better guidance + proc = run_command(cmd, cwd=self.working_dir, capture=True, check=False) + + if proc.returncode != 0: + output = (proc.stderr or "") + (proc.stdout or "") + self._handle_apply_error(output) + success("Terraform applied successfully") + def _handle_apply_error(self, output: str) -> None: + """Handle terraform apply errors with actionable guidance.""" + # Check for known transient errors + if "no matching ELB found" in output or "no matching EC2 Load Balancer found" in output: + warn("\n" + "=" * 70) + warn("NLB not yet provisioned by AWS.") + warn("This is expected on first run - AWS needs 2-5 minutes to create the NLB.") + warn("\nTo retry:") + warn(" 1. Wait 2-3 minutes for AWS to provision the NLB") + warn(" 2. Rerun the deploy command") + warn("=" * 70 + "\n") + fail("Terraform apply failed: NLB not ready (rerun after waiting)") + + if "Error acquiring the state lock" in output: + warn("\n" + "=" * 70) + warn("Terraform state is locked.") + warn("This may happen if a previous run was interrupted.") + warn("\nTo resolve:") + warn(" 1. Verify no other terraform process is running") + warn(" 2. If safe, run: terraform force-unlock ") + warn("=" * 70 + "\n") + fail("Terraform apply failed: state locked") + + # Generic error - print output and fail + fail(f"Terraform apply failed:\n{output}") + def destroy( self, var_args: list[str] | None = None, auto_approve: bool = True, refresh: bool = False ) -> None: diff --git a/tools/validation.py b/tools/validation.py index ab5bef4..0ac58a1 100644 --- a/tools/validation.py +++ b/tools/validation.py @@ -178,6 +178,85 @@ def wait_for_loadbalancer_and_api( return "" # Unreachable, but satisfies type checker +def wait_for_ingress_and_api( + ingress_hostname: str, + api_path: str = "/v1", + retries: int = 60, + interval: int = 10, +) -> str: + """ + Wait for Ingress endpoint to be healthy (HTTPS). + + Args: + ingress_hostname: Full hostname (e.g., vfn-01.testnet.scratchpad.movementnetwork.xyz) + api_path: API path to check (default: /v1) + retries: Number of retries + interval: Interval between retries in seconds + + Returns: + Ingress hostname on success + + Raises: + SystemExit: If timeout or API fails health check + """ + import ssl + import urllib.request + + info(f"Waiting for Ingress endpoint to be healthy: https://{ingress_hostname}") + info("Note: DNS propagation and TLS cert issuance may take 2-10 minutes on first deploy") + + # Create SSL context that verifies certificates + ssl_context = ssl.create_default_context() + last_error = "" + + for attempt in range(1, retries + 1): + url = f"https://{ingress_hostname}{api_path}" + try: + req = urllib.request.Request(url, headers={"Host": ingress_hostname}) + with urllib.request.urlopen(req, timeout=10, context=ssl_context) as resp: + body = resp.read().decode("utf-8", errors="replace") + if resp.status == 200: + payload = json.loads(body) + ledger_version = str(payload.get("ledger_version", "")) + if ledger_version.isdigit(): + info(f"✅ Ingress API healthy at {ingress_hostname} (ledger_version={ledger_version})") + return ingress_hostname + except urllib.error.HTTPError as e: + last_error = f"HTTP {e.code}" + warn(f"Ingress reachable, API HTTP {e.code}, retry {attempt}/{retries}") + except ssl.SSLError as e: + last_error = f"TLS: {e}" + warn(f"TLS not ready ({e}), retry {attempt}/{retries}") + except urllib.error.URLError as e: + last_error = f"Connection: {e.reason}" + if "Name or service not known" in str(e.reason): + warn(f"DNS not propagated yet, retry {attempt}/{retries}") + else: + warn(f"Connection failed ({e.reason}), retry {attempt}/{retries}") + except Exception as e: + last_error = str(e) + warn(f"Ingress/API not ready ({str(e)}), retry {attempt}/{retries}") + + time.sleep(interval) + + # Provide actionable guidance on timeout + warn("\n" + "=" * 70) + warn(f"Timeout waiting for ingress endpoint: https://{ingress_hostname}") + warn(f"Last error: {last_error}") + warn("\nPossible causes:") + warn(" - DNS record not yet propagated (wait 2-5 minutes, then retry)") + warn(" - TLS certificate not yet issued (check: kubectl get certificate -A)") + warn(" - Ingress controller not ready (check: kubectl get pods -n ingress-nginx)") + warn(" - Pod not ready (check: kubectl get pods -n )") + warn("\nTo debug:") + warn(f" kubectl get ingress -A") + warn(f" kubectl describe certificate wildcard-tls -n ingress-nginx") + warn(f" nslookup {ingress_hostname}") + warn("=" * 70 + "\n") + fail("Ingress validation failed - see guidance above") + return "" # Unreachable, but satisfies type checker + + def validate_deployment( namespace: str, service_name: str, @@ -186,6 +265,8 @@ def validate_deployment( interval: int = 10, validate_api: bool = True, kubeconfig_path: Path | None = None, + ingress_enabled: bool = False, + ingress_hostname: str | None = None, ) -> None: """ Validate that a deployment is healthy. @@ -198,6 +279,8 @@ def validate_deployment( interval: Check interval in seconds validate_api: Whether to validate LoadBalancer and API health (default: True) kubeconfig_path: Optional path to kubeconfig + ingress_enabled: Whether ingress mode is used (skips LB check, uses HTTPS) + ingress_hostname: Full ingress hostname for API check (required if ingress_enabled) Raises: SystemExit: If validation fails @@ -213,15 +296,24 @@ def validate_deployment( kubeconfig_path=kubeconfig_path, ) - # Optionally wait for LoadBalancer and API health + # Validate API endpoint if validate_api: - wait_for_loadbalancer_and_api( - namespace=namespace, - service_name=service_name, - retries=lb_retries, - interval=interval, - kubeconfig_path=kubeconfig_path, - ) + if ingress_enabled: + if not ingress_hostname: + fail("ingress_hostname required when ingress_enabled=True") + wait_for_ingress_and_api( + ingress_hostname=ingress_hostname, + retries=lb_retries, + interval=interval, + ) + else: + wait_for_loadbalancer_and_api( + namespace=namespace, + service_name=service_name, + retries=lb_retries, + interval=interval, + kubeconfig_path=kubeconfig_path, + ) info("✅ Deployment validation passed")