From bb5ec58463defb99d9ae0d1c58b529ee2cbb2096 Mon Sep 17 00:00:00 2001 From: Cody Hill Date: Wed, 2 Sep 2026 15:45:05 -0500 Subject: [PATCH] strongswan-ipsec: Azure-optimal defaults, automatic VAES kernel, preflight play; add bandwidth-test One command now deploys a correctly performing IPsec gateway against a managed cloud peer. The customer edits two files - inventory.ini and group_vars/all.yml (five values) - and site.yml does the rest: preflight, kernel, gateways, clients. Defaults (were single-tunnel / conservative, now tuned for Azure/GCP/AWS): * gcmaes256 IKE+ESP; one tunnel per peer address (vpn_tunnel_count 0 = auto, so listing an Azure active-active gateway's two IPs yields two tunnels) * IKE identities derived from ansible_host (the public IP the peer matches on) * stateless datapath: conntrack off, no SNAT, so multi-gateway ECMP survives a failover; host tuning on; RFS off (measured: 176x fewer retransmits) * clients ECMP across every host in the vpn_gateways group automatically Kernel play (tasks/kernel.yml): installs a VAES-capable kernel (6.11) where the CPU has vaes and the bound AES-GCM driver does not - measured +23% per tunnel (4.67 -> 5.75 Gbps) and +16..50% on the 1-5 gateway curve. Verifies the initramfs was actually built and the GRUB default (incl. pinned GRUB_DEFAULT / saved_entry) before spending a reboot. Reboots only a host with no VPN on it yet; a live gateway gets the kernel installed and a note (vpn_kernel_reboot_live opts in). Never reboots a host already on the requested kernel, so it cannot loop. Refuses 7.0 by documentation: same crypto, measured 2.4x slower overall. Preflight play (tasks/preflight.yml) replaces scripts/preflight.sh and gates everything with any_errors_fatal: the five values (checked before any SSH), apt-based OS with reachable mirrors (retried past the unattended-upgrades lock), required kernel modules, uplink MTU >= 1500, every client inside a gateway's vpn_client_cidr, ansible_host is an IPv4 literal, and UDP 500/4500 in BOTH directions between gateways in the run (skipped, with the reason, for a cloud peer, an established SA, or a peer excluded by --limit). Skipped for --tags teardown so a broken deployment stays removable. Datapath fixes: stale xfrm devices and detached ip rules purged (by pref) when the tunnel count shrinks; health timers forget their memo whenever the route is reinstalled; IP identities never indexed; replay_window validated 0-4096 (32768 fails to install). Probe state lives root-only under /run with O_NOFOLLOW writes; pkill patterns audited so none can match its own shell. bandwidth-test/: new sibling solution that measures at the RECEIVER, one iperf3 process per flow, so it works through a managed gateway on either end. Fixes from review: summary play sees bw_results_dir, stale servers killed, client JSON cleared (iperf3 --logfile appends), directories 0755. Docs: DEPLOYING-10G.md (measured 1-5 gateway curve on both kernels, what Azure does NOT do by default and why it costs 3.3x, where the bottleneck sits before and after), README rewritten for the two-file flow with corrected defaults table, root README entries for bandwidth-test and site-to-site-vpn (the latter was failing the structure check before this change). Tested on Crusoe eu-iceland2-a <-> eu-norway1-a, 20 client pairs per side: 1/2/3/4/5 gateways = 2.37/5.60/9.42/16.16/20.66 Gbps on 6.11 (2.05/4.38/7.38/ 10.78/14.79 on 6.8); one 8 vCPU VM with two tunnels = 8.46 Gbps; kernel play verified to leave a live gateway on 6.8 un-rebooted and to reboot it into 6.11 with vaes when asked; UDP probe verified PASS both ways and skipped correctly under --limit; bandwidth-test verified end to end through the tunnel. Reviewed by 8 independent reviewers + 3 skeptics per finding: 49 confirmed findings fixed, all re-tested. Not validated: a real Azure peer, BGP/FRR against a real peer, the "route" transport. --- README.md | 10 +- bandwidth-test/.gitignore | 10 + bandwidth-test/README.md | 121 +++ bandwidth-test/ansible/ansible.cfg | 9 + bandwidth-test/ansible/group_vars/all.yml | 11 + bandwidth-test/ansible/inventory.ini | 24 + .../roles/bandwidth_test/defaults/main.yml | 65 ++ .../roles/bandwidth_test/tasks/main.yml | 195 ++++ .../bandwidth_test/templates/flows.json.j2 | 48 + bandwidth-test/ansible/site.yml | 67 ++ bandwidth-test/scripts/aggregate.py | 89 ++ strongswan-ipsec/.gitignore | 6 + strongswan-ipsec/AZURE-10G-GUIDE.md | 347 +++++++ strongswan-ipsec/README.md | 746 ++++++++++---- strongswan-ipsec/ansible/group_vars/all.yml | 124 +++ .../ansible/group_vars/remote_clients.yml | 14 - .../ansible/group_vars/source_clients.yml | 16 - .../ansible/group_vars/vpn_gateways.yml | 8 - .../ansible/host_vars/gw-2.yml.example | 58 ++ strongswan-ipsec/ansible/host_vars/gw-rmt.yml | 30 - strongswan-ipsec/ansible/host_vars/gw-src.yml | 27 - strongswan-ipsec/ansible/inventory.ini | 59 +- .../roles/vpn_client/defaults/main.yml | 113 ++- .../ansible/roles/vpn_client/tasks/main.yml | 342 +++++-- .../vpn_client/templates/client-paths.json.j2 | 39 + .../templates/client-perf-tuning.sh.j2 | 32 + .../templates/sysctl-vpn-client.conf.j2 | 40 + .../templates/vpn-client-health.service.j2 | 8 + .../templates/vpn-client-health.sh.j2 | 50 + .../templates/vpn-client-health.timer.j2 | 11 + .../templates/vpn-client.service.j2 | 26 +- .../vpn_client/templates/vpn-client.sh.j2 | 130 +++ .../roles/vpn_gateway/defaults/main.yml | 536 +++++++++- .../roles/vpn_gateway/handlers/main.yml | 6 + .../roles/vpn_gateway/tasks/kernel.yml | 394 +++++++ .../ansible/roles/vpn_gateway/tasks/main.yml | 958 ++++++++++++++---- .../roles/vpn_gateway/tasks/preflight.yml | 411 ++++++++ .../templates/ecmp-health.service.j2 | 8 + .../vpn_gateway/templates/ecmp-health.sh.j2 | 75 ++ .../templates/ecmp-health.timer.j2 | 11 + .../vpn_gateway/templates/frr-daemons.j2 | 23 + .../roles/vpn_gateway/templates/frr.conf.j2 | 58 ++ .../vpn_gateway/templates/perf-tuning.sh.j2 | 171 ++++ .../templates/preflight-probe.sh.j2 | 147 +++ .../templates/strongswan-vpn.conf.j2 | 10 + .../vpn_gateway/templates/swanctl-vpn.conf.j2 | 61 +- .../vpn_gateway/templates/sysctl-vpn.conf.j2 | 48 +- .../vpn_gateway/templates/tunnels.json.j2 | 105 ++ .../templates/vpn-network.service.j2 | 46 +- .../vpn_gateway/templates/vpn-network.sh.j2 | 335 ++++++ strongswan-ipsec/ansible/site.yml | 78 +- strongswan-ipsec/k8s/vpn-client.yaml | 252 +++-- 52 files changed, 5859 insertions(+), 749 deletions(-) create mode 100644 bandwidth-test/.gitignore create mode 100644 bandwidth-test/README.md create mode 100644 bandwidth-test/ansible/ansible.cfg create mode 100644 bandwidth-test/ansible/group_vars/all.yml create mode 100644 bandwidth-test/ansible/inventory.ini create mode 100644 bandwidth-test/ansible/roles/bandwidth_test/defaults/main.yml create mode 100644 bandwidth-test/ansible/roles/bandwidth_test/tasks/main.yml create mode 100644 bandwidth-test/ansible/roles/bandwidth_test/templates/flows.json.j2 create mode 100644 bandwidth-test/ansible/site.yml create mode 100755 bandwidth-test/scripts/aggregate.py create mode 100644 strongswan-ipsec/AZURE-10G-GUIDE.md create mode 100644 strongswan-ipsec/ansible/group_vars/all.yml delete mode 100644 strongswan-ipsec/ansible/group_vars/remote_clients.yml delete mode 100644 strongswan-ipsec/ansible/group_vars/source_clients.yml delete mode 100644 strongswan-ipsec/ansible/group_vars/vpn_gateways.yml create mode 100644 strongswan-ipsec/ansible/host_vars/gw-2.yml.example delete mode 100644 strongswan-ipsec/ansible/host_vars/gw-rmt.yml delete mode 100644 strongswan-ipsec/ansible/host_vars/gw-src.yml create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/client-paths.json.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/client-perf-tuning.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/sysctl-vpn-client.conf.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.service.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.timer.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/tasks/kernel.yml create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/tasks/preflight.yml create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.service.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.timer.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr-daemons.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr.conf.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/perf-tuning.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/preflight-probe.sh.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/strongswan-vpn.conf.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/tunnels.json.j2 create mode 100644 strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.sh.j2 diff --git a/README.md b/README.md index 69e7a9f..9cc8a66 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,10 @@ Lets pods on Crusoe Managed Kubernetes assume AWS IAM roles directly via IRSA, u ### Networking +[Bandwidth Test](./bandwidth-test/) + +An Ansible-driven iperf3 harness that generates traffic from one-or-many source hosts to one-or-many destination hosts and collects the results from the receivers, so it measures correctly even when the path in between is a managed appliance you don't control — an Azure VPN Gateway, a GCP HA VPN, a customer firewall. Supports paired, many-to-one and full-mesh topologies, TCP or UDP, and a public-IP baseline mode for comparing a tunnel against the raw path. + [/etc/hosts Pin](./etchosts-pin/README.md) A daemon that resolves a hostname on a fixed interval and keeps the resulting A/AAAA records in `/etc/hosts`. Works around undesirable TTL cache values from intermediate DNS resolvers @@ -165,9 +169,13 @@ A daemon that resolves a hostname on a fixed interval and keeps the resulting A/ A Helm chart that establishes a highly-available IPSec VPN between a remote site and a CMK cluster using paired StrongSwan deployments with BGP-based dynamic route sharing, so that pod, node, and service IPs on both sides are mutually reachable. Includes an example Terraform module for standing up a matching Google Cloud VPN endpoint. +[Crusoe Site-to-Site VPN (AWS / GCP)](./site-to-site-vpn/) + +A hardened, redundant route-based IPsec (IKEv2) VPN terminating on one or two Ubuntu VMs running strongSwan and FRR, with BGP dynamic routing and automatic tunnel failover, provisioned entirely by Terraform from a single params file. Pairs with AWS Site-to-Site VPN or GCP HA VPN; the GCP path and dual-VM HA mode are validated end to end. + [StrongSwan Site-to-Site VPN for Crusoe Cloud](./strongswan-ipsec/) -An Ansible-managed, encrypted IPsec site-to-site VPN between a Crusoe Cloud region and a remote site — another Crusoe region, or Azure/GCP/AWS — with VMs on both sides communicating via their real (non-NAT'd) IP addresses. Uses GRE-over-FOU on the Crusoe side to work around SDN port-security source-IP checks, while the remote cloud side relies on native IP-forwarding; supports adding VMs incrementally via inventory changes. +An Ansible-managed, encrypted IPsec site-to-site VPN between a Crusoe Cloud region and a remote site — another Crusoe region, or Azure/GCP/AWS — with VMs on both sides communicating via their real (non-NAT'd) IP addresses over a GRE-over-FOU overlay or plain routes. You fill in an inventory and five values, and one command preflights connectivity, installs a VAES-capable kernel where it pays, and configures gateways and clients with defaults tuned for a managed cloud peer. Measured at 2.4 Gbps with one gateway per side rising to 20.7 Gbps with five, and 8.5 Gbps through a single VM holding two tunnels; also configures managed Kubernetes nodes via a DaemonSet. ## Contributing diff --git a/bandwidth-test/.gitignore b/bandwidth-test/.gitignore new file mode 100644 index 0000000..9a1fe04 --- /dev/null +++ b/bandwidth-test/.gitignore @@ -0,0 +1,10 @@ +# Fetched iperf3 results land here (bw_results_dir, relative to ansible/). +# They are per-run measurements, not part of the solution. +ansible/results/ +results/ + +# Ansible +*.retry + +# Editor/OS +.DS_Store diff --git a/bandwidth-test/README.md b/bandwidth-test/README.md new file mode 100644 index 0000000..17e4d9f --- /dev/null +++ b/bandwidth-test/README.md @@ -0,0 +1,121 @@ +# Bandwidth Test + +Point-to-point and many-to-many throughput measurement with iperf3, driven by +Ansible. Generates traffic from one-or-many source hosts to one-or-many +destination hosts and **collects the results from the servers**. + +Use it to measure a VPN tunnel, a peering link, a managed cloud gateway +(Azure ↔ GCP), or just two VMs in the same subnet for a baseline. It does not +care what is in the middle. + +## Why server-side collection + +The receiving end is the authoritative measurement — it counts what actually +arrived. It is also the only end that still reports usefully when the path in +between is an appliance you do not control: a managed VPN gateway, a customer +firewall, a carrier link. If the path degrades badly enough, a client's own +summary can be truncated or lost; the server's file is already on disk. + +Every flow gets its own port and its own one-shot server (`iperf3 -s -1`), so +each result is a clean, complete JSON object rather than an appended stream. + +## Quick start + +```bash +cd ansible +vim inventory.ini # source_hosts and dest_hosts, with bw_private_ip +ansible-playbook -i inventory.ini site.yml +``` + +Output: + +``` +====================================================================== +BANDWIDTH TEST - 20 flow(s) measured at the receiver +====================================================================== + flow Gbps secs retrans + 000-src-1-to-dst-1 2.11 30.0 412 + 001-src-2-to-dst-2 2.08 30.0 380 + ... +---------------------------------------------------------------------- + TOTAL 42.41 Gbps across 20 flow(s) + mean per flow 2.12 Gbps + total retransmits 26,417,060 +``` + +## Topologies + +| `bw_mode` | Meaning | +|---|---| +| `paired` (default) | `source[i]` → `dest[i]`. Groups must be the same size. | +| `many_to_one` | every source → `dest[0]`. Load-tests one receiver. | +| `mesh` | every source → every destination. Flows = sources × destinations. | + +```bash +ansible-playbook -i inventory.ini site.yml -e bw_mode=many_to_one +``` + +## Common runs + +```bash +# baseline: straight over the public IPs, bypassing whatever you are testing +ansible-playbook -i inventory.ini site.yml -e bw_dest_address=public + +# UDP, to separate packet loss from TCP's reaction to it +ansible-playbook -i inventory.ini site.yml -e bw_protocol=udp -e bw_udp_bitrate=2G + +# the other direction +ansible-playbook -i inventory.ini site.yml -e bw_reverse=true + +# longer, more streams +ansible-playbook -i inventory.ini site.yml -e bw_duration=120 -e bw_streams=16 +``` + +## Reading the results + +- **Run a baseline first** (`bw_dest_address=public`). Without it you cannot + tell "the tunnel is slow" from "the path is slow" or "the hosts are small". +- **Never judge on a single stream.** One TCP flow pins to one path and one + CPU core by design. Use `bw_streams` ≥ 8, or several flows. +- **Watch retransmits, not just Gbps.** A large retransmit count means the + path is being driven past its knee. Throughput can look good while the link + is thrashing. +- **If TCP is slow but UDP is clean**, you are looking at MTU, MSS, or an + offload problem, not capacity. Re-run with `bw_protocol=udp` at a few packet + sizes to confirm. +- **Compare like with like.** Stream count and duration both change the + number; keep them fixed across the runs you intend to compare. + +## Requirements + +- SSH access to every host, with `become` for the package install +- `iperf3` (installed automatically unless `bw_install: false`) +- The chosen port range open between sources and destinations: + `bw_port_base` .. `bw_port_base + flows - 1` (default 5201+) +- Python 3 on the controller for the summary + +## Tunables + +Everything lives in `ansible/roles/bandwidth_test/defaults/main.yml`: + +| Variable | Default | Purpose | +|---|---|---| +| `bw_mode` | `paired` | `paired`, `many_to_one`, `mesh` | +| `bw_dest_address` | `private` | `private` through the path, `public` for a baseline | +| `bw_duration` | `30` | seconds per flow | +| `bw_streams` | `8` | parallel TCP streams per flow | +| `bw_omit` | `5` | seconds of slow start discarded | +| `bw_protocol` | `tcp` | `tcp` or `udp` | +| `bw_udp_bitrate` | `1G` | per flow, UDP only | +| `bw_reverse` | `false` | measure destination → source | +| `bw_port_base` | `5201` | first port; one per flow | +| `bw_results_dir` | `results` | where fetched JSON lands | +| `bw_install` | `true` | install iperf3 if missing | + +## Limitations + +- One flow per source/destination pair per run. For more concurrency per pair, + raise `bw_streams` or use `mesh`. +- No traffic shaping or scheduling — every flow starts at once. +- Results are raw iperf3 JSON. The aggregator summarises; it does not chart. +- IPv4 only, as written. diff --git a/bandwidth-test/ansible/ansible.cfg b/bandwidth-test/ansible/ansible.cfg new file mode 100644 index 0000000..0fd5234 --- /dev/null +++ b/bandwidth-test/ansible/ansible.cfg @@ -0,0 +1,9 @@ +[defaults] +roles_path = roles +host_key_checking = False +forks = 50 +gathering = smart + +[ssh_connection] +pipelining = True +ssh_args = -o ControlMaster=auto -o ControlPersist=300s -o StrictHostKeyChecking=no diff --git a/bandwidth-test/ansible/group_vars/all.yml b/bandwidth-test/ansible/group_vars/all.yml new file mode 100644 index 0000000..2678127 --- /dev/null +++ b/bandwidth-test/ansible/group_vars/all.yml @@ -0,0 +1,11 @@ +--- +# The inventory groups are FIXED: [source_hosts] and [dest_hosts]. Ansible +# resolves play targets before variables load, so site.yml names them +# literally and they cannot be renamed here. + +# See roles/bandwidth_test/defaults/main.yml for every tunable. +# bw_mode: "paired" # paired | many_to_one | mesh +# bw_duration: 30 +# bw_streams: 8 +# bw_protocol: "tcp" # tcp | udp +# bw_dest_address: "private" diff --git a/bandwidth-test/ansible/inventory.ini b/bandwidth-test/ansible/inventory.ini new file mode 100644 index 0000000..9b859fb --- /dev/null +++ b/bandwidth-test/ansible/inventory.ini @@ -0,0 +1,24 @@ +# =========================================================================== +# Bandwidth test inventory +# +# Two groups. Sources generate traffic, destinations receive and measure it. +# They can be anywhere - either side of a VPN, either side of a managed +# gateway, or the same subnet for a baseline. +# +# bw_private_ip is the address the sources should connect to (the one that +# routes through the path you want to measure). ansible_host is used only for +# SSH, and for the "public" baseline mode. +# =========================================================================== + +[source_hosts] +src-1 ansible_host= bw_private_ip= +src-2 ansible_host= bw_private_ip= + +[dest_hosts] +dst-1 ansible_host= bw_private_ip= +dst-2 ansible_host= bw_private_ip= + +[all:vars] +ansible_user=ubuntu +ansible_become=true +ansible_ssh_common_args=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null diff --git a/bandwidth-test/ansible/roles/bandwidth_test/defaults/main.yml b/bandwidth-test/ansible/roles/bandwidth_test/defaults/main.yml new file mode 100644 index 0000000..5a600ea --- /dev/null +++ b/bandwidth-test/ansible/roles/bandwidth_test/defaults/main.yml @@ -0,0 +1,65 @@ +--- +# =========================================================================== +# bandwidth_test role +# +# Generates iperf3 traffic from one-or-many source hosts to one-or-many +# destination hosts and collects the results FROM THE SERVERS. +# +# Server-side collection is the point. The receiving end is the authoritative +# measurement, and it is the only end that still reports usefully when the +# path in between is a managed appliance you do not control - an Azure VPN +# Gateway, a GCP HA VPN, a customer firewall. A client's own summary can be +# lost or truncated when the path degrades; the server's is not. +# =========================================================================== + +# --------------------------------------------------------------------------- +# What talks to what +# --------------------------------------------------------------------------- +# paired source[i] -> dest[i] (equal-length groups; the default) +# many_to_one every source -> dest[0] +# mesh every source -> every dest (sources x dests flows) +bw_mode: "paired" + +# Which address on each destination the sources connect to. +# private the destination's private IP (through the tunnel / VPN) +# public the destination's public IP (direct, for a baseline) +# Set bw_dest_ip on a host to override it explicitly. +bw_dest_address: "private" + +# --------------------------------------------------------------------------- +# Test parameters +# --------------------------------------------------------------------------- +bw_duration: 30 +bw_streams: 8 +bw_omit: 5 # seconds of slow start to discard +bw_protocol: "tcp" # tcp | udp +bw_udp_bitrate: "1G" # per flow, udp only +bw_reverse: false # measure destination -> source instead +bw_port_base: 5201 # each flow gets its own port: base + index + +# --------------------------------------------------------------------------- +# Inventory groups +# --------------------------------------------------------------------------- +# The role enumerates sources and destinations from these two groups. They are +# NOT free to rename: site.yml targets its plays with the literal group names +# [source_hosts] and [dest_hosts], because Ansible resolves play targets before +# any variable is loaded. Change all three together or not at all. +bw_source_group: "source_hosts" +bw_dest_group: "dest_hosts" + +# --------------------------------------------------------------------------- +# Collection +# --------------------------------------------------------------------------- +# Each server runs one-shot ("iperf3 -s -1") and writes its own JSON, so the +# file ON THE SERVER is the measurement. Results are fetched to bw_results_dir. +bw_remote_dir: "/tmp/bandwidth-test" +# Absolute on purpose: fetch resolves a relative dest against the playbook +# directory, but the Summarise play runs a command from wherever you are. +bw_results_dir: "{{ playbook_dir }}/results" + +# Extra seconds to wait past bw_duration before giving up on a server. +bw_grace: 45 + +# Install iperf3 if missing. Set false if the image already has it, or if you +# have no package access on these hosts. +bw_install: true diff --git a/bandwidth-test/ansible/roles/bandwidth_test/tasks/main.yml b/bandwidth-test/ansible/roles/bandwidth_test/tasks/main.yml new file mode 100644 index 0000000..6c4eb7c --- /dev/null +++ b/bandwidth-test/ansible/roles/bandwidth_test/tasks/main.yml @@ -0,0 +1,195 @@ +--- +# ============================================================================= +# bandwidth_test role +# +# Two passes, driven by bw_phase: +# servers start a one-shot iperf3 server per flow on the destination hosts +# clients run the matching iperf3 clients on the source hosts +# collect fetch each server's JSON - the authoritative measurement +# +# site.yml drives all three in order. +# ============================================================================= + +- name: Install iperf3 + ansible.builtin.package: + name: iperf3 + state: present + become: true + when: bw_install | bool + tags: [always] + +- name: Build the flow list + ansible.builtin.set_fact: + bw_flows: "{{ lookup('template', 'flows.json.j2') | from_json }}" + run_once: true + delegate_to: localhost + become: false + tags: [always] + +- name: Share the flow list with every host + ansible.builtin.set_fact: + bw_flows: "{{ hostvars[groups['all'][0]].bw_flows | default(bw_flows) }}" + tags: [always] + +- name: Fail if no flows were produced + ansible.builtin.assert: + that: bw_flows | length > 0 + fail_msg: >- + No flows. Check that {{ bw_source_group }} and {{ bw_dest_group }} both + exist in the inventory and are non-empty. In "paired" mode they must have + the same number of hosts. + run_once: true + tags: [always] + +# --------------------------------------------------------------------------- +# Servers - one one-shot iperf3 per flow, each writing its own JSON +# --------------------------------------------------------------------------- + +- name: Prepare the results directory + ansible.builtin.file: + path: "{{ bw_remote_dir }}" + state: directory + mode: "0755" + become: true + when: bw_phase == 'servers' + tags: [servers] + +# A server left over from an interrupted run would hold its port, and because +# iperf3 --logfile appends, its eventual JSON would land in the new file. +- name: Stop iperf3 servers left over from an interrupted run + ansible.builtin.shell: "pkill -x iperf3 || true" + changed_when: false + become: true + when: bw_phase == 'servers' + tags: [servers] + +- name: Clear previous results + ansible.builtin.shell: "rm -f {{ bw_remote_dir }}/*.json" + changed_when: true + become: true + when: bw_phase == 'servers' + tags: [servers] + +# "-1" makes the server handle exactly one test and exit, so the JSON it +# writes is a complete single result rather than an appended stream. +# No "disown" here: Ansible's shell module runs /bin/sh, which on Debian and +# Ubuntu is dash, and dash has no disown builtin. nohup plus a full set of +# redirections is enough to detach. +- name: Start a one-shot iperf3 server per inbound flow + ansible.builtin.shell: | + nohup iperf3 --server --one-off --port {{ item.port }} --json \ + --logfile {{ bw_remote_dir }}/{{ item.file }} \ + /dev/null 2>&1 & + loop: "{{ bw_flows | selectattr('dest', 'equalto', inventory_hostname) | list }}" + loop_control: + label: "port {{ item.port }} for {{ item.source }}" + changed_when: true + when: bw_phase == 'servers' + tags: [servers] + +- name: Wait for the servers to listen + ansible.builtin.shell: | + for i in $(seq 1 20); do + ss -ltn 2>/dev/null | grep -q ':{{ item.port }} ' && exit 0 + sleep 0.5 + done + exit 1 + loop: "{{ bw_flows | selectattr('dest', 'equalto', inventory_hostname) | list }}" + loop_control: + label: "port {{ item.port }}" + changed_when: false + when: bw_phase == 'servers' + tags: [servers] + +# --------------------------------------------------------------------------- +# Clients +# --------------------------------------------------------------------------- + +- name: Prepare the client results directory + ansible.builtin.file: + path: "{{ bw_remote_dir }}" + state: directory + mode: "0755" + become: true + when: bw_phase == 'clients' + tags: [clients] + +# iperf3 --logfile APPENDS. A second run would leave two JSON documents in one +# file and the retransmit column would silently go missing. +- name: Clear previous client results + ansible.builtin.shell: "rm -f {{ bw_remote_dir }}/client-*.json" + changed_when: true + become: true + when: bw_phase == 'clients' + tags: [clients] + +# The client writes its own JSON too. The SERVER remains the authoritative +# throughput measurement, but retransmits are a sender-side counter that the +# receiver cannot see, so the client file supplies that half. +- name: Run iperf3 + ansible.builtin.shell: >- + iperf3 --client {{ item.dest_ip }} --port {{ item.port }} + --time {{ bw_duration }} --omit {{ bw_omit }} + {% if bw_protocol == 'udp' %}--udp --bitrate {{ bw_udp_bitrate }}{% else %}--parallel {{ bw_streams }}{% endif %} + {% if bw_reverse | bool %}--reverse{% endif %} + --connect-timeout 10000 + --json --logfile {{ bw_remote_dir }}/client-{{ item.file }} + loop: "{{ bw_flows | selectattr('source', 'equalto', inventory_hostname) | list }}" + loop_control: + label: "{{ item.source }} -> {{ item.dest }} ({{ item.dest_ip }}:{{ item.port }})" + register: bw_client_runs + changed_when: true + failed_when: false + async: "{{ (bw_duration | int) + (bw_grace | int) }}" + poll: 0 + when: bw_phase == 'clients' + tags: [clients] + +- name: Wait for the clients to finish + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + register: bw_wait + until: bw_wait.finished + retries: "{{ ((bw_duration | int) + (bw_grace | int)) // 5 }}" + delay: 5 + loop: "{{ bw_client_runs.results | default([]) }}" + loop_control: + label: "{{ item.item.source | default('?') }} -> {{ item.item.dest | default('?') }}" + when: bw_phase == 'clients' + failed_when: false + tags: [clients] + +# --------------------------------------------------------------------------- +# Collect - the server's own JSON is the measurement +# --------------------------------------------------------------------------- + +- name: Fetch the server-side results (authoritative) + ansible.builtin.fetch: + src: "{{ bw_remote_dir }}/{{ item.file }}" + dest: "{{ bw_results_dir }}/{{ item.file }}" + flat: true + fail_on_missing: false + loop: "{{ bw_flows | selectattr('dest', 'equalto', inventory_hostname) | list }}" + loop_control: + label: "{{ item.file }}" + when: bw_phase == 'collect' + tags: [collect] + +- name: Fetch the client-side results (for retransmits) + ansible.builtin.fetch: + src: "{{ bw_remote_dir }}/client-{{ item.file }}" + dest: "{{ bw_results_dir }}/client-{{ item.file }}" + flat: true + fail_on_missing: false + loop: "{{ bw_flows | selectattr('source', 'equalto', inventory_hostname) | list }}" + loop_control: + label: "client-{{ item.file }}" + when: bw_phase == 'collect_clients' + tags: [collect] + +- name: "Teardown: stop any iperf3 left running" + ansible.builtin.shell: "pkill -x iperf3 || true" + changed_when: false + become: true + when: bw_phase == 'collect' + tags: [collect] diff --git a/bandwidth-test/ansible/roles/bandwidth_test/templates/flows.json.j2 b/bandwidth-test/ansible/roles/bandwidth_test/templates/flows.json.j2 new file mode 100644 index 0000000..3e20286 --- /dev/null +++ b/bandwidth-test/ansible/roles/bandwidth_test/templates/flows.json.j2 @@ -0,0 +1,48 @@ +{#- + Build the flow list: one entry per source -> destination pair, each with its + own TCP/UDP port so every flow gets its own one-shot iperf3 server and its + own result file. + + Derived once on the controller and reused by both plays, so the source and + the destination always agree on ports. +-#} +{%- set srcs = groups[bw_source_group] | default([]) -%} +{%- set dests = groups[bw_dest_group] | default([]) -%} +{%- set pairs = [] -%} +{%- if bw_mode == 'paired' -%} + {%- for i in range([srcs | length, dests | length] | min) -%} + {%- set _ = pairs.append((srcs[i], dests[i])) -%} + {%- endfor -%} +{%- elif bw_mode == 'many_to_one' -%} + {%- for s in srcs -%} + {%- set _ = pairs.append((s, dests[0])) -%} + {%- endfor -%} +{%- else -%} + {%- for s in srcs -%} + {%- for d in dests -%} + {%- set _ = pairs.append((s, d)) -%} + {%- endfor -%} + {%- endfor -%} +{%- endif -%} +{%- set flows = [] -%} +{%- for p in pairs -%} + {%- set src = p[0] -%} + {%- set dst = p[1] -%} + {%- set hv = hostvars[dst] -%} + {#- explicit override, else the requested address family -#} + {%- if hv.bw_dest_ip is defined and hv.bw_dest_ip -%} + {%- set ip = hv.bw_dest_ip -%} + {%- elif bw_dest_address == 'public' -%} + {%- set ip = hv.ansible_host | default(hv.ansible_default_ipv4.address) -%} + {%- else -%} + {%- set ip = hv.bw_private_ip | default(hv.ansible_default_ipv4.address) -%} + {%- endif -%} + {%- set _ = flows.append({ + 'index': loop.index0, + 'source': src, + 'dest': dst, + 'dest_ip': ip, + 'port': (bw_port_base | int) + loop.index0, + 'file': 'flow-%03d-%s-to-%s.json' | format(loop.index0, src, dst) }) -%} +{%- endfor -%} +{{ flows | to_json }} diff --git a/bandwidth-test/ansible/site.yml b/bandwidth-test/ansible/site.yml new file mode 100644 index 0000000..3f4ac71 --- /dev/null +++ b/bandwidth-test/ansible/site.yml @@ -0,0 +1,67 @@ +--- +# Bandwidth test: start one-shot iperf3 servers, run the clients, then collect +# each SERVER's own JSON. Three ordered plays because the servers must be +# listening before any client connects. +# +# The inventory MUST define the groups [source_hosts] and [dest_hosts] - +# Ansible resolves play targets before group_vars are loaded, so these cannot +# be variables. +# +# ansible-playbook -i inventory.ini site.yml +# ansible-playbook -i inventory.ini site.yml -e bw_protocol=udp -e bw_udp_bitrate=2G +# ansible-playbook -i inventory.ini site.yml -e bw_mode=many_to_one +# ansible-playbook -i inventory.ini site.yml -e bw_dest_address=public # baseline + +- name: Start iperf3 servers on the destinations + hosts: dest_hosts + gather_facts: true + roles: + - role: bandwidth_test + vars: + bw_phase: servers + +- name: Generate traffic from the sources + hosts: source_hosts + gather_facts: true + roles: + - role: bandwidth_test + vars: + bw_phase: clients + +- name: Collect results from the destinations + hosts: dest_hosts + gather_facts: false + roles: + - role: bandwidth_test + vars: + bw_phase: collect + +- name: Collect client-side results from the sources + hosts: source_hosts + gather_facts: false + roles: + - role: bandwidth_test + vars: + bw_phase: collect_clients + +- name: Summarise + hosts: localhost + gather_facts: false + connection: local + # The inventory's [all:vars] usually sets ansible_become for the remote + # hosts. That is a host variable, which beats the play-level "become" + # keyword, so it has to be overridden as a variable here. + vars: + ansible_become: false + # Implicit localhost is in no group and includes no role, so it sees + # neither group_vars nor the role default. Same value as the role's, so + # the fetch destination and the aggregate input are one directory. + bw_results_dir: "{{ playbook_dir }}/results" + tasks: + - name: Aggregate the server-side results + ansible.builtin.command: "python3 {{ playbook_dir }}/../scripts/aggregate.py {{ bw_results_dir }}" + register: bw_summary + changed_when: false + - name: Report + ansible.builtin.debug: + msg: "{{ bw_summary.stdout_lines }}" diff --git a/bandwidth-test/scripts/aggregate.py b/bandwidth-test/scripts/aggregate.py new file mode 100755 index 0000000..938ba61 --- /dev/null +++ b/bandwidth-test/scripts/aggregate.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Summarise the server-side iperf3 results. + +Reads the JSON each iperf3 SERVER wrote for its own flow. The receiver is the +authoritative measurement - it counts what actually arrived, which is what you +want when the path in between is a managed gateway you do not control. + + aggregate.py [results-dir] +""" +import glob, json, os, sys + +d = sys.argv[1] if len(sys.argv) > 1 else "results" +files = sorted(f for f in glob.glob(os.path.join(d, "flow-*.json")) + if not os.path.basename(f).startswith("client-")) +if not files: + sys.exit(f"no results in {d}/ - did the servers run?") + +# retransmits are a sender-side counter, so they come from the client's file +# when one was collected; throughput always comes from the server's +client = {} +for cf in glob.glob(os.path.join(d, "client-flow-*.json")): + try: + cj = json.load(open(cf)) + key = os.path.basename(cf)[len("client-"):-5] + client[key] = (cj.get("end", {}).get("sum_sent") or {}).get("retransmits") + except Exception: + pass + +rows, bad = [], [] +for f in files: + name = os.path.basename(f)[:-5] + try: + j = json.load(open(f)) + except Exception as e: + bad.append(f"{name}: unreadable ({e})") + continue + if "error" in j: + bad.append(f"{name}: {j['error']}") + continue + end = j.get("end", {}) + # UDP reports under sum; TCP under sum_received on the server side + s = end.get("sum_received") or end.get("sum") or {} + if not s: + bad.append(f"{name}: no summary in the server's JSON") + continue + rows.append({ + "flow": name.replace("flow-", ""), + "gbps": s.get("bits_per_second", 0) / 1e9, + "seconds": s.get("seconds", 0), + "retrans": client.get(os.path.basename(f)[:-5]), + "lost_pct": s.get("lost_percent"), + "jitter_ms": s.get("jitter_ms"), + }) + +rows.sort(key=lambda r: r["flow"]) +udp = any(r["lost_pct"] is not None for r in rows) + +print("=" * 74) +print(f"BANDWIDTH TEST - {len(rows)} flow(s) measured at the receiver") +print("=" * 74) +hdr = f" {'flow':<34}{'Gbps':>8}{'secs':>7}" +hdr += f"{'loss %':>9}{'jitter ms':>11}" if udp else f"{'retrans':>10}" +print(hdr) +for r in rows: + line = f" {r['flow'][:34]:<34}{r['gbps']:>8.2f}{r['seconds']:>7.1f}" + if udp: + line += f"{(r['lost_pct'] or 0):>9.2f}{(r['jitter_ms'] or 0):>11.3f}" + else: + line += f"{r['retrans'] if r['retrans'] is not None else 'n/a':>10}" + print(line) + +total = sum(r["gbps"] for r in rows) +print("-" * 74) +print(f" {'TOTAL':<34}{total:>8.2f} Gbps across {len(rows)} flow(s)") +if rows: + print(f" {'mean per flow':<34}{total/len(rows):>8.2f} Gbps") +if not udp: + rt = [r["retrans"] for r in rows if r["retrans"] is not None] + if rt: + print(f" {'total retransmits':<34}{sum(rt):>8,}") + else: + print(" retransmits: n/a (no client-side results collected)") + print(" Retransmits are the efficiency signal: a large number means the") + print(" path is being driven past its knee, even if throughput looks good.") +if bad: + print() + print(f" {len(bad)} flow(s) produced no result:") + for b in bad: + print(f" {b}") diff --git a/strongswan-ipsec/.gitignore b/strongswan-ipsec/.gitignore index ceb2b98..60b1d6c 100644 --- a/strongswan-ipsec/.gitignore +++ b/strongswan-ipsec/.gitignore @@ -1 +1,7 @@ CLAUDE.md + +# Ansible +*.retry + +# Editor/OS +.DS_Store diff --git a/strongswan-ipsec/AZURE-10G-GUIDE.md b/strongswan-ipsec/AZURE-10G-GUIDE.md new file mode 100644 index 0000000..8850480 --- /dev/null +++ b/strongswan-ipsec/AZURE-10G-GUIDE.md @@ -0,0 +1,347 @@ +# Reaching 10 Gbps to Azure with N+1 + +A worked deployment against **Azure VPN Gateway**: how many Crusoe gateway VMs +you need, what the playbook already handles, and — the part that bites people — +**what Azure does not enable for you.** + +For the solution itself — transports, crypto profiles, variables, +troubleshooting — see [README.md](README.md). §4 here (sizing, kernel, host +knobs) applies to **any** peer; §3.1 is the Azure-specific part. + +Everything labelled **measured** was measured on Crusoe between +`eu-iceland2-a` and `eu-norway1-a`, 54 ms RTT, on `c` series VMs (AMD EPYC +9655P, Zen 5, BlueField-3 VF). **Derived** is arithmetic on those numbers, not +an observation. **Published** comes from the cloud vendor's own documentation. + +--- + +## 1. The short answer + +| Peer | Gbps per VM | VMs for 10 Gbps | With N+1 | Basis | +|---|---|---|---|---| +| **Azure** VpnGw5 | 4.6 | 3 | **4** | published | +| **Crusoe ⇄ Crusoe**, `c` series + kernel 6.11 | 2.4 → 4.1 | 4 | **5** | **measured** | + +Two thresholds are worth separating, because the work involved is very +different: + +- **4.6 Gbps** — one gateway VM, `ansible-playbook -i inventory.ini site.yml`, + and **two** non-default settings on the Azure side: active-active mode, for + the second public IP that becomes the second tunnel, and the cipher policy + (§3.1 items 2 and 3). This is the easy win. +- **10 Gbps** — three gateway VMs against Azure, and Azure needs **BGP plus + active-active mode plus a Local Network Gateway per VM**. None of that can + be automated from this repo. That is what §3 is for. + +### The measured Crusoe-to-Crusoe curve + +20 client pairs per side, GRE-over-FOU overlay, `gcmaes256`, 54 ms path. +Deployed and measured with the shipped playbook, on both kernels: + +| gateways/side | kernel 6.8 | **kernel 6.11** | change | per gateway | peak core | +|---|---|---|---|---|---| +| 1 | 2.05Gbps | **2.37Gbps** | +15.5% | 2.37Gbps | 89% | +| 2 | 4.38Gbps | **5.60Gbps** | +27.8% | 2.80Gbps | 63% | +| 3 | 7.38Gbps | **9.42Gbps** | +27.7% | 3.14Gbps | 93% | +| 4 | 10.78Gbps | **16.16Gbps** | +49.9% | 4.04Gbps | 92% | +| 5 | 14.79Gbps | **20.66Gbps** | +39.7% | 4.13Gbps | 92% | + +**Four gateways per side clears 10 Gbps** — three gives 9.42, just short. Lose +one of four and you are left with 9.42, which is 94% of target; deploy five if +you need a hard 10 Gbps floor through a single failure. + +Note that **throughput per gateway rises with gateway count** (2.37 → 4.13). +That is not noise. With one gateway per side the same box receives every +client's FOU flow *and* terminates the tunnel, so both directions contend on +one machine. Adding gateways relieves the client-facing side, and the +per-gateway figure converges on the pure per-tunnel limit. + +This is also why a *derived* number was not good enough here. Arithmetic on +the single-tunnel figure (2 × 5.75 Gbps, NIC-bounded) predicted roughly 8 Gbps +per VM and 2 VMs for 10 Gbps. The measurement says 2.4–4.1 Gbps per VM and 4 +VMs. The GRE leg costs far more than the arithmetic suggested. + +--- + +## 2. What actually limits throughput + +In the order they bite. Know which ceiling you are against before tuning +anything. + +| Limit | Value | Basis | +|---|---|---| +| Azure, per **tunnel** (GCMAES256) | 2.3 Gbps | published | +| Azure, per **tunnel** (default CBC policy) | 0.7 Gbps | published | +| Azure, per gateway aggregate (VpnGw5) | 10 Gbps | published | +| Tunnels per VM against a cloud peer | 2 | published | +| One tunnel, one core — kernel 6.11 | 5.75 Gbps | measured | +| One tunnel, one core — kernel 6.8 | 4.67 Gbps | measured | +| **One gateway VM, 2 tunnels, GRE overlay** | **8.46 Gbps** | **measured** | +| One **outer flow** to the receiver's NIC | 8.1 Gbps | measured | +| Whole WAN path, 16 separate flows | 9.0 Gbps | measured | + +**One tunnel is one outer 4-tuple, so one RX queue, so one core.** Nothing +software-side splits it. More cores per VM does not help; more tunnels does. + +**Every user packet crosses a GRE gateway's NIC twice** — inbound as +GRE-over-FOU from the client, outbound as ESP to the peer. A gateway needs +roughly 2 Gbps of NIC for every 1 Gbps of user traffic. + +--- + +## 3. What is NOT a default — the manual work + +### 3.1 On the Azure side. None of this can be automated from here. + +| # | Do this | Azure default? | Why it matters | +|---|---|---|---| +| 1 | SKU **VpnGw5** (or VpnGw5AZ — zone-redundant, same speed) | no, you pick it | 10 Gbps aggregate. VpnGw4 caps at 5. | +| 2 | Enable **active-active** mode | **NO — off** | Gives the gateway two public IPs. That is what turns one Crusoe VM into two tunnels instead of one. | +| 3 | Custom **IPsec/IKE policy** on **every** Connection | **NO — off** | The single highest-value setting on this page. See below. | +| 4 | One **Local Network Gateway per Crusoe gateway VM**, each carrying that VM's public IP and your Crusoe CIDR | no | Azure matches an on-premises device by its public IP. | +| 5 | **Enable BGP** on the gateway and every LNG — unique ASN and peer address per LNG | **NO — off** | Azure will not spread return traffic across several on-premises devices without it. Needed only above one gateway VM. | +| 6 | Shared key on every Connection = your `vpn_psk` | no | Must be byte-identical. | +| 7 | One **Connection per LNG**, so 3 VMs = 3 Connections = 6 tunnels | no | 6 × 2.3 = 13.8 Gbps of capacity into a 10 Gbps SKU. | + +**Item 3 in full, because it is worth 3.3×.** Azure's default policy +negotiates AES-CBC with HMAC-SHA, which its own tables rate at **700 Mbps per +tunnel**. GCMAES256 is rated **2.3 Gbps**. On each Connection set: + +| Field | Value | +|---|---| +| IPsec Encryption | **GCMAES256** | +| IPsec Integrity | **GCMAES256** | +| IKE Encryption | AES256 | +| IKE Integrity | SHA384 | +| DH Group | ECP384 | +| PFS Group | ECP384 | +| SA lifetime | 3600 s (matches this role's `rekey_time`) | + +Our side already proposes exactly this (`vpn_crypto_profile: "gcmaes256"` is +the default, negotiating `aes256gcm16-ecp384`). If Azure is left on its +default policy, IKE either fails with `NO_PROPOSAL_CHOSEN` or — worse — +succeeds at 700 Mbps and looks fine. + +```bash +# Verify what actually got negotiated, on any gateway: +sudo swanctl --list-sas | grep -o 'ESP:[^ ]*' +# want: ESP:AES_GCM_16-256 +``` + +### 3.2 On the Crusoe side + +| # | Do this | Why | +|---|---|---| +| 1 | Create **3 gateway VMs** for 10 Gbps, **4** for N+1. 8 vCPU each. | One VM is 4.6 Gbps. Do not make them bigger — see §4.1. | +| 2 | Open the **VPC firewall**: UDP 500 + 4500 between your gateway public IPs and Azure's; UDP 9473 inside the VPC; ICMP inside the VPC | 9473 is the GRE-over-FOU overlay. ICMP is the client health probe that removes a dead gateway from the ECMP set. | +| 3 | List every gateway VM in `inventory.ini` | Clients ECMP across the whole `vpn_gateways` group automatically — no list to maintain. | +| 4 | Set `vpn_bgp_enabled: true` plus `vpn_bgp_local_asn` and `vpn_bgp_peer_addrs` per gateway | Required by Azure item 5, and only above one gateway VM. | +| 5 | Open UDP 500 + 4500 on **both** firewalls before you deploy | The **preflight play** probes UDP 500/4500 both ways between Crusoe gateways you own, but against Azure there is nothing to run a listener on — so the proof is the SA establishing, and an IKE timeout there almost always means one direction is blocked. Preflight still checks MTU, kernel modules, apt reachability, and that every client VM is inside `vpn_client_cidr`. | + +### 3.3 What you do NOT need to touch + +All of this is already the default. Changing it will most likely make things +worse — the measurements are in §4.3. + +`gcmaes256` · one tunnel per peer address (so 2 against active-active) · UDP +encapsulation forced · GRE-over-FOU to your VMs · GRO off on the uplink · +MSS derived from the real path MTU · RFS off · ECMP L4 hashing on gateways +*and* clients · loose reverse-path filtering · conntrack disabled with both +its interlocks · anti-replay window 1024 · NIC IRQ pinning and XPS · +performance governor · client TCP buffers and BBR · a VAES-capable kernel +installed and booted if the running one predates 6.11. + +--- + +## 4. Sizing and tuning + +### 4.1 Instance size: 8 vCPU, not larger + +Against a cloud peer you get two tunnels per VM, each pinning one core for +inbound decrypt. On these VMs 8 vCPU is **four physical cores** with SMT — +`thread_siblings_list` reads `0-1`, `2-3`, `4-5`, `6-7` — so two tunnels use +half the real cores and leave the rest for the GRE leg. + +One tunnel at 5.75 Gbps put its decrypt core at **62%**. Growing the instance +does not move that number, because the work cannot leave that core. + +### 4.2 Kernel: the largest single host-side win + +The kernel binds the highest-priority driver registered for +`rfc4106(gcm(aes))`. Kernels before 6.11 ship only the 2010-era AES-NI +implementation. + +| Kernel | Bound AES-GCM driver | Mean | Decrypt core | Crypto share of RX cycles | +|---|---|---|---|---| +| 6.8.0-78 | `rfc4106-gcm-aesni` | 4.67 Gbps | 83% | 22.6% | +| **6.11.0-29** | **`rfc4106-gcm-vaes-avx10_512`** | **5.75 Gbps** | **62%** | **under 3%** | +| 7.0.0-30 | `rfc4106-gcm-vaes-avx512` | 2.16 Gbps | 83% | 1.6% | + +**+23% per tunnel**, and the playbook does it for you — install, initramfs, +GRUB, reboot, and a post-reboot assertion that the fast driver is actually +bound. On a first deploy it runs before any VPN config exists, so the reboot +interrupts nothing. On a gateway that already carries tunnels it installs the +kernel and **does not reboot** — it prints a note, and you reboot in a window +one gateway at a time, or pass `-e vpn_kernel_reboot_live=true`. It never +reboots a host that is already running the requested kernel, so it cannot +loop. + +**Do not point `vpn_kernel_package` at 7.0.** It carries the same fast crypto +and still measured **2.4× slower** end to end. A `perf` profile explains it: +on 6.8 one symbol dominates at 17% (`key_256_dec_update`, the AES-NI decrypt), +while on 7.0 no symbol exceeds 2.5% and the cycles spread across +`fib_rules_lookup`, `nft_meta_store_ifname`, `xfrm_sk_policy_lookup`, +`pskb_expand_head` and the `srso_safe_ret` speculation mitigation — which 6.11 +reports as `Not affected` on the same CPU. The crypto got cheap; everything +else got expensive. + +### 4.3 Host knobs, measured + +The one that matters is **RFS off**, and it is already the default. + +Receive Flow Steering sends a packet to the CPU where the consuming socket +last ran. For a single IPsec tunnel that hands one SA's packets between CPUs, +so they reach `xfrm_input` out of order and the ESP anti-replay window rejects +the stragglers. The tunnel stays `INSTALLED`, ping is clean, and only TCP +suffers. + +| `rps_flow_cnt` | Throughput | Retransmits | Anti-replay drops | +|---|---|---|---| +| 4096 | 4.67 Gbps | 18,624 | 46,697 | +| **0** | **4.86 Gbps** | **106** | **3,141** | + ++4.2% and **176× fewer retransmits**. Read honestly: those figures are at +4.7 Gbps on kernel 6.8. On 6.11 at 5.7 Gbps the rejections come back to +roughly 1M per 20 s with RFS *already off*, because at that rate the +reordering is the WAN path's, not the host's — the no-tunnel control on the +same path retransmits 2.57M at 9 Gbps. RFS-off removes the reordering *this +host* causes. It cannot fix the network. + +Everything else was tested and left alone: + +| Change | Effect | Why | +|---|---|---| +| `ethtool -G rx 8192 tx 8192` | **−9.1%** | A deeper ring holds more in flight, widening reordering | +| `rx_cqe_moder` + `rx_cqe_compress` | −0.2% | Nothing. Widely recommended; does not apply here | +| `ethtool -C rx-usecs 128` | −2.0% | Over-batched | +| `ethtool -C rx-usecs 64` | +2.6% | The only positive one, and small | +| `gro_normal_batch 64` | +1.1% | Inside this rig's noise band | +| `esp-hw-offload` | n/a | `off [fixed]` — impossible on a VF; a platform-team ask | +| `replay_window 32768` | breaks | SA never installs, tunnel never comes up | +| More vCPU per gateway | 0% | One tunnel cannot leave one core | + +--- + +## 5. N+1 + +Deploy `N+1` gateways per side and let ECMP spread flows across all of them. +Lose one and the remaining `N` still carry the target. + +This only works if a return packet may arrive on a **different** gateway than +the one that sent it — so no SNAT and no conntrack. That is what +`vpn_disable_conntrack: true` and its two interlocks enforce, and the role +fails the play if you break them. + +| Failure injected | Throughput after | Expected (4/5) | +|---|---|---| +| One tunnel killed | 84% | 80% | +| One gateway VM killed | 79% | 80% | + +Measured on a 5-gateway deployment. Both land where the arithmetic says. +Expect a brief reconvergence, not a clean step. + +--- + +## 6. Verify it + +Preflight is a play, not a script, and it gates every other play. To run just +that part: + +```bash +ansible-playbook -i inventory.ini site.yml --tags preflight +``` + +It checks the five required values, an apt-based OS with reachable mirrors, the +kernel modules the datapath needs, a 1500-byte uplink, that every client VM +falls inside some gateway's `vpn_client_cidr`, and UDP 500 + 4500 in **both** +directions between gateways. + +After deploying, the five things worth looking at: + +```bash +sudo swanctl --list-sas | grep -o 'ESP:[^ ]*' # want AES_GCM_16-256 +sed -n "/^name .*: rfc4106(gcm(aes))$/,+1p" /proc/crypto # want a *vaes* driver +cat /sys/class/net/ens3/queues/rx-0/rps_flow_cnt # want 0 +grep XfrmInStateSeqError /proc/net/xfrm_stat # want flat, not climbing +ethtool -k ens3 | grep generic-receive-offload # want off +``` + +`XfrmInStateSeqError` climbing means packets are arriving out of order and +anti-replay is discarding them. Check `rps_flow_cnt` first. + +**Measure with one process per flow.** A single `iperf3` server process is +single-threaded and caps out around 4–5 Gbps, which is very easy to mistake +for a network limit. The companion `bandwidth-test` solution collects at the +receiver, one process per flow, so it works through a managed gateway on +either end. + +--- + +## 7. Where the bottleneck sits, before and after + +**Before this work, the Crusoe side was the bottleneck.** A gateway delivered +roughly 1.5–2 Gbps per tunnel, against an Azure per-tunnel allowance of 2.3. +Tuning it was not optional — we were the constraint, and no amount of Azure +configuration would have helped. + +**After the changes in §4** — kernel 6.11, RFS off, GRO off on the uplink, the +right cipher — one tunnel carries **5.75 Gbps** (measured). That clears +Azure's 2.3 Gbps per-tunnel allowance with room to spare, so the ceiling has +moved off our side and onto Azure's. + +The whole-VM figure was measured too, in exactly the shape Azure presents: +**one 8 vCPU gateway VM holding two tunnels to two distinct peer public IPs +carried 8.46 Gbps** of real client traffic over the GRE overlay, with both +tunnels active (the ECMP hash split them 41% / 59% across 24 flows — even +distribution needs more flows than that). + +So the same VM that Azure will cap at **4.6 Gbps** is demonstrably good for +**8.46**. That is the gap the sizing in §1 rests on, and it is why adding host +tuning past this point cannot reduce the Azure VM count. + +That is what makes the sizing arithmetic in §1 trustworthy: + +- Azure caps **2.3 Gbps per tunnel**, and one on-premises device gets **2 + tunnels** (published) +- So **4.6 Gbps per VM**, and `ceil(10 / 4.6)` = **3 VMs**, N+1 = **4** + +From here, further host tuning will not reduce that VM count, because the +limit is no longer ours to move. Host tuning buys you fewer VMs only where the +far end has **no per-tunnel cap**: Crusoe to Crusoe, or a peer that terminates +IPsec on general-purpose compute. + +One thing would put us back under the cap: **leaving Azure on its default +cipher policy** (§3.1 item 3). At 700 Mbps per tunnel that is 1.4 Gbps per VM, +and 10 Gbps would need **8 VMs** instead of 3. + +--- + +## 8. What was not measured + +Stated plainly, so nobody builds on sand. + +- **Every Azure number here is published, not measured.** No Azure peer was + tested. +- **BGP and FRR were never brought up against a real peer.** Since 10 Gbps + through Azure requires BGP (§3.1 item 5), that path needs validating on + first use. +- **The `route` transport was never validated end to end.** Crusoe port + security was on throughout, and a foreign-source packet was confirmed + dropped by the fabric. `gre_fou` is the tested path and the default. +- The Crusoe-to-Crusoe curve in §1 **is measured**, on the shipped playbook, + 1 through 5 gateways per side. An earlier *derived* estimate in this + document claimed ~8 Gbps per VM and 2 VMs for 10 Gbps; the measurement + replaced it with 2.4–4.1 Gbps per VM and 4 VMs. Do not trust arithmetic on + single-tunnel figures for a GRE deployment. +- Single-tunnel figures are **gateway to gateway**, pure IPsec with no GRE + leg. Real client traffic through the overlay will be lower. diff --git a/strongswan-ipsec/README.md b/strongswan-ipsec/README.md index 5d96f6b..f0c17a8 100644 --- a/strongswan-ipsec/README.md +++ b/strongswan-ipsec/README.md @@ -1,17 +1,138 @@ # StrongSwan Site-to-Site VPN for Crusoe Cloud -## Overview - -Encrypted IPsec VPN between a Crusoe Cloud region and a remote site. The -remote site can be a **customer datacenter**, **another Crusoe region**, or -**Azure**, **GCP**, or **AWS**. Supports both standalone VMs (Ansible) and -managed Kubernetes clusters (DaemonSet). - -**Key capabilities:** -- Subnet-to-subnet routing over encrypted IPsec tunnel -- Full tunnel mode — all internet traffic exits via the remote site's firewall -- Managed K8s support — DaemonSet configures nodes automatically, no SSH needed -- GRE-over-FOU overlay bypasses Crusoe's port_security and firewall restrictions +Encrypted IPsec VPN between a Crusoe Cloud region and a remote site — a +customer datacenter, another Crusoe region, or Azure, GCP or AWS. Configures +standalone VMs with Ansible and managed Kubernetes nodes with a DaemonSet. + +**Capabilities** + +- Subnet-to-subnet routing over encrypted IPsec, plus full-tunnel mode +- Managed K8s support — the DaemonSet configures nodes, no SSH needed +- **Selectable client transport** — GRE-over-FOU overlay (default) or plain + static routes +- **Named crypto profiles**, including `gcmaes256` +- **Multiple tunnels per gateway** with ECMP, for peers that publish more than + one outer address — most managed cloud VPNs do in their redundant mode +- **Multiple gateway VMs** with client-side ECMP, for throughput beyond one + tunnel. BGP (FRR) is included, and managed cloud VPNs generally require it + before they will spread return traffic across several on-premises devices; + between two sites you control, static routes are enough. + +**The defaults target a managed cloud VPN peer** (Azure, GCP, AWS) **and have +changed** from earlier +versions of this solution: GCMAES256 for IKE and ESP, one tunnel per peer +address, a stateless datapath (conntrack off, no SNAT), host tuning on, and a +VAES-capable kernel installed — and, on a host with no VPN yet, booted — +automatically. To pin the old behaviour set `vpn_crypto_profile: default`, +`vpn_tunnel_count: 1`, `vpn_disable_conntrack: false`, +`vpn_stateful_forward_rule: true`, `vpn_perf_tuning: false` and +`vpn_kernel_upgrade: false`. + +To measure any of this, use the separate +[bandwidth-test](../bandwidth-test/) solution. It drives iperf3 across +one-or-many hosts and collects results from the receivers, so it works through +a managed gateway on either end. + +## Measured throughput + +Iceland ⇄ Norway across the public internet (54 ms RTT), 20 client pairs, +GRE-over-FOU, `gcmaes256`, dedicated 8 vCPU gateway VMs: + +| gateways/site | tunnels/gateway | kernel 6.8 | **kernel 6.11** | change | per gateway | +|---|---|---|---|---|---| +| 1 | 1 | 2.05 | **2.37** | +15.5% | 2.37 | +| 2 | 1 | 4.38 | **5.60** | +27.8% | 2.80 | +| 3 | 1 | 7.38 | **9.42** | +27.7% | 3.14 | +| 4 | 1 | 10.78 | **16.16** | +49.9% | 4.04 | +| 5 | 1 | 14.79 | **20.66** | +39.7% | 4.13 | +| 5 | 5 (full mesh) | 42.4 | not re-run | — | 8.5 | + +Both columns were deployed and measured with the shipped playbook. Kernel 6.11 +is worth +15% to +50% for a package install — see below. + +Measured separately, in the shape a managed cloud peer presents: **one 8 vCPU +gateway VM holding two tunnels to two distinct peer public IPs carried +8.46 Gbps** of client traffic, with both tunnels active (ECMP split them +41% / 59% across 24 flows). + +Note that a managed cloud VPN will usually cap a tunnel well below what this +side can drive, so the peer — not Crusoe — sets your VM count. Work that out +from your provider's published per-tunnel figure; the arithmetic for Azure is +in [AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md). + +**Four gateways per side clears 10 Gbps**; three gives 9.42, just short. + +Throughput scales with the **number of tunnels**, because each tunnel's +inbound decrypt runs on its own single CPU core. Per-gateway throughput also +*rises* with gateway count (2.37 → 4.13): with one gateway per side the same +box receives every client's FOU flow and terminates the tunnel, so both +directions contend on one machine. Nothing reached a link ceiling — 42.4 Gbps +on the mesh was still CPU-bound. + +Failover: killing one of five gateways mid-transfer degraded throughput to +**84%** (strongSwan stopped) and **79%** (all forwarding dropped) of before — +both matching the expected 4/5 — and recovered on restore. Proportional +degradation, not a blackhole. + +### Per-tunnel throughput, and the kernel + +Measured separately, gateway to gateway (pure IPsec, no GRE leg), one tunnel, +`c` series VMs (AMD EPYC 9655P), 54 ms path, with a 9.01 Gbps no-tunnel +control repeated either side of the change: + +| kernel | bound AES-GCM driver | mean | peak | decrypt core | +|---|---|---|---|---| +| 6.8.0-78 | `rfc4106-gcm-aesni` | 4.67 | 5.18 | 83% | +| **6.11.0-29** | **`rfc4106-gcm-vaes-avx10_512`** | **5.75** | **6.07** | **62%** | +| 7.0.0-30 | `rfc4106-gcm-vaes-avx512` | 2.16 | 2.23 | 83% | + +Kernel 6.11 is worth **+23% per tunnel** for a package install: it is the first +release with a VAES/AVX-512 AES-GCM implementation, which drops crypto from +22.6% of receive cycles to under 3%. Kernel 7.0 carries the same fast crypto +and still measured 2.4× slower overall, so do not assume newer is better — +measure. `vpn_check_crypto_accel` reports which driver you are actually on. + +Two other measured limits worth knowing before you size anything: + +- A **single outer flow** delivers at most **8.1 Gbps** to the receiver's NIC + across this WAN path, flat across offered rates from 3 to 9 Gbps. +- A single `iperf3` **server process** is single-threaded and caps out around + 4–5 Gbps. Use one process per flow or you will measure your test tool. + +**[AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md)** works a full 10 Gbps deployment +end to end against Azure: the VM-count arithmetic, every host knob that was +measured (including the ones that made things worse), the settings Azure does +not enable for you, and where the bottleneck sits before and after. §4 there — +instance size, the kernel, and the host knobs — applies to **any** peer; only +§3.1 is Azure-specific. + +## Sizing + +**8 vCPU per gateway is the right default.** Each tunnel's decrypt occupies one +core, and against a managed cloud VPN you typically get **two tunnels per +gateway** — a redundant cloud VPN gateway publishes two outer addresses, and +one on-premises device gets one tunnel to each. Two decrypt cores plus 3–5 +lightly loaded cores for the client-facing side fits comfortably in 8, with +headroom. + +Scale throughput by **adding gateway VMs**, not by growing them. Each gateway +pair is worth roughly 2–3 Gbps on kernel 6.8; see +[AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md) for the kernel that improves that and +for a worked VM count to reach 10 Gbps with N+1 redundancy. + +Two caveats worth knowing: + +- With **one** tunnel per gateway, a 16 vCPU instance measured no faster than 8 + — it pinned a single core at 94% and left the rest idle. Verified in both + traffic directions. +- With **many** tunnels per gateway (a Crusoe-to-Crusoe mesh, which no cloud + peer will give you), cores do start to matter: at 5 tunnels each, 8 vCPU + saturated all 8 cores while 16 vCPU still had headroom. Size for roughly one + core per terminating tunnel plus 3–5 for the client side. + +Gateways sustained 15–18 Gbps of NIC traffic each, so the NIC is rarely the +limit. Note that every user packet crosses a gateway's NIC **twice** — inbound +as GRE-over-FOU, outbound as ESP — so NIC load is about 2× the user traffic. ## Architecture @@ -23,178 +144,372 @@ managed Kubernetes clusters (DaemonSet). | K8s Nodes | GRE-over-FOU | VMs / services | | (DaemonSet) |----+ | (datacenter) | | Standalone VMs | | (UDP 9473) | | - | (Ansible) | | +--------+-----------+ - +--------------------+ | | + | (Ansible) | | or plain +--------+-----------+ + +--------------------+ | routing | v | +------------+ IPsec +------------+ - | gw-src |============| gw-rmt | - | GRE + FOU | encrypted | direct fwd | + | gw-1 |============| peer | + | xfrm0..N | encrypted | direct fwd | | (Crusoe) | (UDP 4500) | + SNAT | +------------+ +------------+ | Firewall / Internet ``` -### How it works - -1. **GRE-over-FOU** wraps GRE inside UDP (port 9473). This bypasses Crusoe's - SDN port_security (which blocks packets with non-matching source IPs) and - works through firewalls that only allow TCP/UDP. - -2. **Gateway (gw-src)** runs a multipoint GRE-over-FOU tunnel with a - pre-populated neighbor table covering the entire client CIDR. New VMs - and K8s nodes are covered automatically. - -3. **Standalone VMs** are configured by the `vpn_client` Ansible role — - point-to-point GRE-over-FOU tunnel to the gateway. - -4. **K8s nodes** are configured by the `vpn-client` DaemonSet (`k8s/vpn-client.yaml`) — - same GRE-over-FOU tunnel, runs as a privileged pod with hostNetwork. - -5. **Mark-based routing** on the gateway ensures all GRE-inbound traffic - is forwarded through xfrm0 into IPsec — even for destinations outside the - datacenter CIDR (needed for full tunnel mode). - -6. **Full tunnel** routes `0.0.0.0/1` + `128.0.0.0/1` through the GRE tunnel. - These are more specific than the default route (`/0`), so they catch all - internet traffic while leaving intra-VPC routes (`/20`, `/24`) unaffected. - The remote gateway SNATs and forwards to the internet. - -### Packet flow +Scaled out, with several gateway VMs: ``` -Crusoe VM / K8s node - → GRE-over-FOU encap (UDP 9473 to Crusoe gateway) - → Crusoe gateway marks packet, routes through xfrm0 - → StrongSwan encrypts via IPsec (ESP, UDP 4500) - → Remote gateway decrypts - → Datacenter traffic: forwards to LAN - → Internet traffic: SNATs to gateway IP, forwards to internet + clients ---ECMP---> gw-1 ==tunnel(s)==> + + gw-2 ==tunnel(s)==> | remote gateway(s) + gw-3 ==tunnel(s)==> + ``` ---- +**How it works** + +1. **Route-based IPsec.** Each tunnel is a CHILD_SA bound to its own `if_id` + with its own XFRM interface (`xfrm0`, `xfrm1`, …). Routing decides what + enters which tunnel. +2. **Client transport** — how local VMs and K8s nodes reach the gateway. See + [Client transports](#client-transports). GRE-over-FOU is the default because + Crusoe's SDN port security blocks a VM from forwarding packets whose source + is not its own address. +3. **Multipoint GRE** on the gateway with a pre-populated neighbour table + covering the client CIDR, so new VMs and nodes are handled automatically. +4. **Mark-based routing** forwards client traffic into IPsec even for + destinations outside the remote CIDR (needed for full tunnel) without + touching the gateway's own default route. +5. **Full tunnel** routes `0.0.0.0/1` + `128.0.0.0/1` through the gateway — + more specific than the default route, so it catches internet traffic while + leaving intra-VPC routes alone. +6. **ECMP, and nothing stateful.** With several tunnels or gateways, traffic is + hashed per flow. No NAT and no connection tracking sit in the path, which is + what allows a reply to return through a *different* tunnel or gateway — + exactly what a cloud VPN gateway does. ## Prerequisites -**Crusoe VPC firewall:** +**Crusoe VPC firewall** | Port | Protocol | Direction | Purpose | |------|----------|-----------|---------| | 500 | UDP | Between gateways (public IPs) | IKE handshake | | 4500 | UDP | Between gateways (public IPs) | IKE NAT-T (all IPsec traffic) | | 9473 | UDP | Within VPC (gateway ↔ VMs/nodes) | GRE-over-FOU | - -**Remote side firewall:** -- UDP 500 + 4500 inbound from Crusoe gateway public IP -- If non-Crusoe: enable NIC-level IP forwarding (Azure/GCP/AWS) - ---- +| 179 | TCP | Between gateway and peer BGP address | BGP, if enabled | +| — | ICMP | Within VPC (clients → gateways) | Client health probe, with >1 gateway | + +**Remote side** + +- UDP 500 + 4500 inbound from every Crusoe gateway public IP +- If the peer is a VM you run: NIC-level IP forwarding (see Cloud + Prerequisites below) + +> **Check both directions before deploying.** IPsec needs UDP 500/4500 each +> way, and a one-way block presents as "IKE timeout" with nothing useful in the +> strongSwan log. Between gateways in your inventory the preflight play tests +> exactly that on every run. Against a managed cloud peer there is nothing to +> run a listener on, so the proof is the tunnel establishing — check both +> firewalls before you deploy. +> +> If clients cannot ICMP their gateways, the multi-gateway health probe cannot +> tell a dead gateway from a live one. It fails safe — leaving routes alone +> rather than removing them — but you lose automatic failover. + +**For `vpn_client_transport: route` only, and no playbook can do it:** Crusoe +must **disable port security** on every gateway VM's vNIC, or add +allowed-address-pairs covering `vpn_remote_subnet`. A gateway in `route` mode +forwards decrypted packets whose source is a *remote* private IP, and the SDN +drops those by default. The symptom is a healthy tunnel with no return traffic. ## Quick Start -### 1. Configure gateways and client VMs +### 1. Configure — two files ```bash cd ansible +vim inventory.ini # WHO: your gateway VM(s) and client VMs +vim group_vars/all.yml # WHAT: five values +``` + +`group_vars/all.yml` asks for exactly five things: + +| Value | What it is | +|---|---| +| `vpn_psk` | `openssl rand -base64 48`, and the same string on the peer (Azure calls it the Connection's *shared key*) | +| `vpn_remote_addrs` | the peer's public IPs — list **every** one to get a tunnel each (a redundant cloud VPN gateway publishes two) | +| `vpn_local_subnet` | your Crusoe CIDR | +| `vpn_remote_subnet` | the peer's CIDR (your cloud VPC/VNet, or the remote site's subnet) | +| `vpn_client_cidr` | which Crusoe IPs join the overlay, usually the same as `vpn_local_subnet` | + +Everything else is already set for a managed cloud peer: GCMAES256, one tunnel +per peer address, GRE-over-FOU to your VMs, a stateless datapath, every +measured host tweak, and a VAES-capable kernel installed and booted if the +running one predates 6.11. The IKE identities derive from `ansible_host`, so +there is nothing to fill in for those. + +> **Encrypt the PSK before committing anything.** Either keep +> `group_vars/all.yml` out of version control, or encrypt just the value: +> +> ```bash +> ansible-vault encrypt_string --stdin-name vpn_psk +> ``` +> +> Run with `--ask-vault-pass`. The role already sets `no_log: true` on the task +> that writes the swanctl config. + +### 2. Deploy -# Gateway IPs and client VMs -vim inventory.ini +```bash +ansible-playbook -i inventory.ini site.yml +``` -# Source gateway (Crusoe) — GRE mode + client CIDR -vim host_vars/gw-src.yml +Four plays run in order, and the first one gates the rest: -# Remote gateway — direct mode (or GRE for Crusoe↔Crusoe) -vim host_vars/gw-rmt.yml +| Play | Does | +|---|---| +| **preflight** | the five values are filled in · the OS is apt-based and its mirrors answer · `xfrm_interface`, `fou`, `ip_gre`, `esp4` exist · uplink MTU ≥ 1500 · every client VM is inside some gateway's `vpn_client_cidr` · **UDP 500 + 4500 pass in both directions between gateways** | +| **kernel** | installs a VAES-capable kernel if the running one predates 6.11, and reboots **only a host with no VPN on it yet** — on a first deploy that is before any config exists. A host already carrying tunnels gets the kernel installed and a note; reboot it in a window, or pass `-e vpn_kernel_reboot_live=true` | +| **gateways** | IPsec, XFRM interfaces, ECMP, firewall, host tuning | +| **clients** | routes from the Crusoe VMs into the overlay | -# Client VM config — which gateway and which subnets to route -vim group_vars/source_clients.yml +Preflight runs with `any_errors_fatal`, so one bad host stops the run before +anything is touched. A half-configured VPN is worse than an unconfigured one. -# PSK (generate with: openssl rand -base64 48) -vim group_vars/vpn_gateways.yml -``` +The bidirectional UDP test is the one that saves the most time: IPsec needs +both ports both ways, and a one-way block presents as "IKE timeout" with +nothing useful in the strongSwan log. It runs only when the peer is a host in +your inventory — a managed cloud gateway has nothing to run a listener on — +and is skipped when a tunnel is already up, because an installed SA is +stronger evidence than any probe. -### 2. Deploy with Ansible +Adding a gateway VM later means adding it to `inventory.ini` and re-running — +clients ECMP across every host in the `vpn_gateways` group automatically, with +no list to maintain. A managed cloud peer will usually need BGP enabled before +it spreads return traffic across them; see `group_vars/all.yml`. + +### 3. K8s nodes ```bash -ansible-playbook -i inventory.ini site.yml +vim ../k8s/vpn-client.yaml # GATEWAY_IPS, TRANSPORT, REMOTE_CIDRS +kubectl apply -f ../k8s/vpn-client.yaml ``` -### 3. Deploy K8s DaemonSet (for managed K8s nodes) +### 4. Verify ```bash -# Edit ConfigMap — set gateway private IP and remote CIDRs -vim k8s/vpn-client.yaml - -# Deploy -kubectl apply -f k8s/vpn-client.yaml +ansible-playbook -i inventory.ini site.yml --tags verify +ssh ping ``` -### 4. Test +To measure throughput, use [bandwidth-test](../bandwidth-test/). + +## Client transports + +`vpn_client_transport` on the gateway; the client role inherits it. + +| Value | Meaning | Prerequisite | +|---|---|---| +| `gre_fou` | GRE-over-FOU overlay (**default**) | none | +| `route` | plain static routes via the gateway's private IP | **port security disabled on each gateway vNIC** | +| `none` | gateway carries no local clients; forwards its own LAN subnet | none | + +`vpn_use_gre: true/false` still works as a deprecated alias. + +`gre_fou` stays the default because `route` needs a manual network change no +playbook can make. Where `route` is allowed it is better: + +| | `gre_fou` | `route` | +|---|---|---| +| Client devices | N GRE devices + FOU sockets | none | +| Client MTU | 1400 | 1400 | +| Gateway setup | FOU module, multipoint GRE, one neighbour entry per host in the CIDR, 2 policy tables | plain FIB forwarding | +| **GRO on the uplink** | **must be OFF** — see troubleshooting | unaffected | +| **Client CIDR size** | a `/20` is 4094 neighbour entries; a `/16` is refused | any size | +| Flow entropy on the fabric | one FOU 4-tuple per client↔gateway pair unless `vpn_fou_sport_auto` | the real client 5-tuples | +| Dead-gateway detection | needs the ICMP probe | `fib_multipath_use_neigh` handles VM death for free | + +On `gre_fou`, set `vpn_fou_sport_auto: true` for better flow spread without the +port-security change — the kernel then hashes the outer FOU source port per +inner flow. + +## Crypto profiles + +| Profile | IKE | ESP | +|---|---|---| +| `default` | `aes256-sha384-ecp256-modp3072` | `aes256gcm128-ecp256-modp3072` | +| **`gcmaes256`** | `aes256gcm16-prfsha384-ecp384` | `aes256gcm16-ecp384` | +| `gcmaes256_fast` | `aes256gcm16-prfsha256-ecp256` | `aes256gcm16-ecp256` | +| `cbc_compat` | `aes256-sha256-modp2048` | `aes256-sha256-modp2048` | + +Setting `vpn_ike_proposals` or `vpn_esp_proposals` explicitly overrides the +profile. + +Two things worth knowing: + +- In strongSwan, `aes256gcm128` and `aes256gcm16` are **the same thing** — + AES-256-GCM with a 128-bit ICV. Azure spells it **GCMAES256**; other clouds + use their own names for the same cipher. So this role's **ESP has always been + GCM**; it was **IKE** that was AES-CBC before `gcmaes256` became the default. +- **AES-GCM is worth several times CBC+HMAC per tunnel** on a managed peer, + because their published per-tunnel rates differ sharply by cipher suite — + Azure's own figures are 2.3 Gbps versus 700 Mbps. Both ends must offer the + same suite or IKE fails with `NO_PROPOSAL_CHOSEN`, and most clouds do **not** + pick GCM by default. See [AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md) §3.1 for + the exact Azure policy fields. + +Verify what was actually negotiated, not what was requested: ```bash -# From a standalone VM -ssh ping +sudo swanctl --list-sas | grep -i aes +``` + +## Multiple tunnels and multiple gateways + +Two independent kinds of fan-out. -# From a K8s node -kubectl exec -n kube-system ds/vpn-client -- ping -c 3 +### Several tunnels from one gateway -# Full tunnel — internet via remote gateway -kubectl exec -n kube-system ds/vpn-client -- ping -c 3 8.8.8.8 +For a peer that publishes more than one outer address. A redundant cloud VPN +gateway has two, which is the practical maximum per on-premises device: + +```yaml +# group_vars/all.yml - nothing else to set +vpn_remote_addrs: + - "" + - "" ``` -### Adding a new Crusoe VM +Two addresses give two tunnels (`vpn_tunnel_count: 0` means one per address) +with IP identities, which is what a managed cloud peer matches on. + +Tunnel *i* gets `xfrm{i}` and `if_id 100+i`, all with identical traffic +selectors, plus one ECMP route across them. + +> **The failure that looks like success:** raising `vpn_tunnel_count` while both +> `vpn_local_addrs` and `vpn_remote_addrs` resolve to a single address. With +> NAT-T both UDP ports are pinned to 4500, so every tunnel shares one outer +> 4-tuple, lands on one receive queue, and performs exactly like one tunnel. +> The role warns when it sees this. + +### Several gateway VMs + +Crusoe gives a VM one public IP, and a managed cloud peer identifies each +on-premises device by a unique public IP, so throughput past one gateway comes +from **more gateway VMs**: ```ini -# Add to inventory.ini: -[source_clients] -vm-s1 ansible_host= -vm-s2 ansible_host= # new +# inventory.ini +[vpn_gateways] +gw-1 ansible_host= +gw-2 ansible_host= +gw-3 ansible_host= ``` -```bash -# Re-run — gateway already covers the CIDR, only the new VM needs config -ansible-playbook -i inventory.ini site.yml +That is the whole change: clients ECMP across every host in `vpn_gateways` +automatically. (A two-site Crusoe deployment needs `vpn_gateway_hosts` set per +site instead — see `host_vars/gw-2.yml.example`.) + +**Against a managed cloud VPN, BGP becomes mandatory.** These services will not +spread return traffic across several on-premises devices without it, and +generally want one peer definition per device with a unique ASN and BGP peer +address. Set `vpn_bgp_enabled: true` plus `vpn_bgp_local_asn`, +`vpn_bgp_advertise`, `vpn_bgp_accept` and `vpn_bgp_peer_addrs`. The exact +objects to create on the Azure side are in +[AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md) §3.1. + +**Between two sites you control, BGP is not needed.** Pair the gateways 1:1 and +give each pair its own tunnel: + +```yaml +# host_vars/site-a-gw-1.yml (and -2, -3 ... paired with the same index) +vpn_local_id: "site-a-gw-1@crusoe.vpn" +vpn_remote_id: "site-b-gw-1@crusoe.vpn" +vpn_remote_gw_ip: "" +vpn_local_subnet: "" +vpn_remote_subnet: "" +vpn_client_cidr: "" ``` -K8s nodes need no changes — the DaemonSet auto-configures new nodes. +For maximum throughput between two sites you own, give every gateway a tunnel +to every peer gateway. `vpn_id_style: indexed` keeps the local identity fixed +and indexes the remote one: ---- +```yaml +vpn_tunnel_count: 5 +vpn_remote_addrs: ["", "", "", "", ""] +vpn_local_id: "site-a-gw-1@crusoe.vpn" +vpn_remote_id: "site-b-gw@crusoe.vpn" # becomes site-b-gw-1@ .. -5@ +vpn_id_style: "indexed" +``` + +That is what produced 42.4 Gbps. No cloud peer will let you do it — it needs +control of both ends. + +### Rules that come with ECMP + +The role enforces these because getting them wrong silently drops traffic: + +- `net.ipv4.fib_multipath_hash_policy = 1` on gateways **and clients**. At the + default `0` the kernel hashes source/destination IP only, so every flow takes + one path and the rest idle. +- **No SNAT and no conntrack in the path.** SNAT is stateful, so a reply + returning through a different gateway has no matching entry. Set + `vpn_disable_conntrack: true`, `vpn_stateful_forward_rule: false`, + `vpn_snat_internet_egress: false`. The role fails if these disagree. +- `rp_filter = 2` (loose). Strict reverse-path filtering rejects the asymmetric + return, which is normal here. ## Full Tunnel Mode Route all internet traffic through the remote site's firewall. -**Standalone VMs** — set in `group_vars/source_clients.yml`: +**Standalone VMs** — `group_vars/all.yml`: ```yaml -vpn_remote_subnets: - - "0.0.0.0/1" - - "128.0.0.0/1" +vpn_remote_subnets: ["0.0.0.0/1", "128.0.0.0/1"] ``` -**K8s nodes** — set in `k8s/vpn-client.yaml` ConfigMap: +**K8s nodes** — `k8s/vpn-client.yaml` ConfigMap: ```yaml REMOTE_CIDRS: "0.0.0.0/1 128.0.0.0/1" ``` -**Gateway (gw-src)** — `vpn_remote_subnet: "0.0.0.0/0"` in `host_vars/gw-src.yml` +**Source gateway** — `vpn_remote_subnet: "0.0.0.0/0"` +**Remote gateway** — `vpn_local_subnet: "0.0.0.0/0"`, plus internet access and +a default route for egress. -**Remote gateway (gw-rmt)** — `vpn_local_subnet: "0.0.0.0/0"` in `host_vars/gw-rmt.yml` -(must also have internet access and a default route for internet egress) +> With full tunnel, VMs are only reachable via a jump host. Set +> `ansible_ssh_common_args` with `ProxyJump` in the inventory. -> **Note**: With full tunnel, VMs are only reachable via jump host (SSH through -> the remote gateway using private IPs). Set `ansible_ssh_common_args` with -> `ProxyJump` in the inventory for ongoing Ansible management. See inventory.ini -> for an example. - ---- +> **Multi-gateway caveat:** full-tunnel internet egress needs SNAT on the +> *remote* gateway, and SNAT is stateful. So the remote side cannot itself be +> ECMP'd across several SNAT gateways. With a managed cloud VPN as the remote +> this is a non-issue — the provider owns its own egress NAT. ## Cloud Prerequisites (Remote Side) -When the remote side is Azure, GCP, or AWS, configure these **before** -running the playbook. When both sides are Crusoe, only VPC firewall rules -are needed (see Prerequisites above). +Whatever the peer, four things have to be true on its side. None of them can be +set from this repo. + +| # | Requirement | Why | +|---|---|---| +| 1 | UDP **500 and 4500** open inbound from every Crusoe gateway public IP | IKE and NAT-T. A one-way block presents as "IKE timeout" with nothing useful in the log | +| 2 | A **route** for your Crusoe CIDR pointing at the peer gateway | otherwise return traffic never leaves | +| 3 | **AES-GCM** offered by the peer's IPsec policy | most managed VPNs default to AES-CBC + HMAC, which their own tables rate several times slower — see [Crypto profiles](#crypto-profiles) | +| 4 | One **unique public IP** per Crusoe gateway VM, and **BGP** if you run more than one | a managed peer identifies each on-premises device by IP, and will not spread return traffic across several without BGP | -### Azure +If your peer is a **managed cloud VPN service** (Azure VPN Gateway, GCP HA VPN, +AWS Site-to-Site VPN), it also imposes a per-tunnel and an aggregate rate cap, +and those caps — not this side — will set your VM count. + +> **Azure**, worked end to end, including the three settings Azure leaves off +> by default and what each costs you: +> **[AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md) §3**. + +If the peer is a **VM you run** (another Crusoe region, a datacenter, an EC2 or +Compute Engine instance), there is no per-tunnel cap and it needs NIC-level IP +forwarding plus a route — the per-cloud commands below. + +### Azure VM as the peer ```bash az network nic update --resource-group --name --ip-forwarding true @@ -208,7 +523,7 @@ az network vnet subnet update \ --resource-group --vnet-name --name --route-table vpn-to-crusoe ``` -### GCP +### GCP VM as the peer ```bash gcloud compute instances create --can-ip-forward ... @@ -217,7 +532,7 @@ gcloud compute routes create vpn-to-crusoe \ --next-hop-instance= --next-hop-instance-zone= ``` -### AWS +### AWS instance as the peer ```bash aws ec2 modify-network-interface-attribute --network-interface-id --no-source-dest-check @@ -225,105 +540,176 @@ aws ec2 create-route --route-table-id \ --destination-cidr-block --network-interface-id ``` ---- - ## File Structure ``` . ├── README.md -├── CLAUDE.md +├── AZURE-10G-GUIDE.md # 10 Gbps N+1 against Azure, worked ├── k8s/ -│ └── vpn-client.yaml # DaemonSet for K8s nodes (GRE-over-FOU) +│ └── vpn-client.yaml # DaemonSet: transport, ECMP, node sysctls └── ansible/ - ├── inventory.ini # Gateways + client VMs - ├── site.yml + ├── inventory.ini # EDIT: who - gateway VMs, client VMs ├── group_vars/ - │ ├── vpn_gateways.yml # PSK - │ ├── source_clients.yml # Source VMs → gw-src - │ └── remote_clients.yml # Remote VMs → gw-rmt (Crusoe↔Crusoe) + │ └── all.yml # EDIT: what - five values + ├── site.yml # check -> kernel -> gateways -> clients ├── host_vars/ - │ ├── gw-src.yml # GRE mode, client CIDR - │ └── gw-rmt.yml # Direct or GRE mode + │ └── gw-2.yml.example # optional, inert: Crusoe<->Crusoe + │ # subnets, or per-gateway BGP └── roles/ - ├── vpn_gateway/ # IPsec + GRE-over-FOU + mark routing + ├── vpn_gateway/ │ ├── defaults/main.yml │ ├── tasks/main.yml - │ ├── templates/ - │ │ ├── swanctl-vpn.conf.j2 - │ │ ├── sysctl-vpn.conf.j2 - │ │ └── vpn-network.service.j2 - │ └── handlers/main.yml - └── vpn_client/ # GRE-over-FOU client (standalone VMs) + │ ├── tasks/preflight.yml # gates everything; must pass first + │ ├── tasks/kernel.yml # VAES kernel: install, boot, verify + │ ├── handlers/main.yml + │ └── templates/ + │ ├── tunnels.json.j2 # derives the tunnel list + │ ├── swanctl-vpn.conf.j2 + │ ├── strongswan-vpn.conf.j2 + │ ├── sysctl-vpn.conf.j2 + │ ├── vpn-network.sh.j2 # ALL links, routes, rules, marks + │ ├── vpn-network.service.j2 + │ ├── ecmp-health.{sh,service,timer}.j2 + │ ├── preflight-probe.sh.j2 # bidirectional UDP 500/4500 probe + │ ├── perf-tuning.sh.j2 + │ └── frr{-daemons,.conf}.j2 # BGP + └── vpn_client/ ├── defaults/main.yml ├── tasks/main.yml - ├── templates/vpn-client.service.j2 - └── handlers/main.yml + └── templates/ + ├── client-paths.json.j2 # derives the gateway path list + ├── vpn-client.{sh,service}.j2 + ├── vpn-client-health.{sh,service,timer}.j2 + ├── sysctl-vpn-client.conf.j2 + └── client-perf-tuning.sh.j2 ``` +`vpn-network.sh` and `vpn-client.sh` are rendered once and called by **both** +the playbook and the systemd unit, so live state and boot state cannot drift. + ## Key Variables -| Variable | Where | Purpose | -|----------|-------|---------| -| `vpn_psk` | `group_vars/vpn_gateways.yml` | IKE pre-shared key | -| `vpn_use_gre` | `host_vars/gw-*.yml` | `true` for Crusoe (GRE-over-FOU), `false` for direct | -| `vpn_client_cidr` | `host_vars/gw-*.yml` | Subnet of VMs/nodes behind this gateway (GRE mode) | -| `vpn_local_subnet` | `host_vars/gw-*.yml` | IPsec traffic selector (local) | -| `vpn_remote_subnet` | `host_vars/gw-*.yml` | IPsec traffic selector (remote, `0.0.0.0/0` for full tunnel) | -| `vpn_remote_gw_ip` | `host_vars/gw-*.yml` | Remote gateway public IP | -| `vpn_gre_key` | `defaults/main.yml` | GRE key (default: 100) | -| `vpn_fou_port` | `defaults/main.yml` | FOU UDP port (default: 9473) | -| `vpn_gateway_host` | `group_vars/*_clients.yml` | Which gateway the VMs connect to | -| `vpn_remote_subnets` | `group_vars/*_clients.yml` | CIDRs to route through VPN (list, for full tunnel) | -| `GATEWAY_IP` | `k8s/vpn-client.yaml` | Gateway private IP (DaemonSet ConfigMap) | -| `REMOTE_CIDRS` | `k8s/vpn-client.yaml` | CIDRs to route through VPN (DaemonSet ConfigMap) | +Gateway (`roles/vpn_gateway/defaults/main.yml` documents every one): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `vpn_psk` | — | IKE pre-shared key | +| `vpn_client_transport` | `""` → `gre_fou`/`none` | `gre_fou`, `route`, or `none` | +| `vpn_client_cidr` | — | subnet of VMs/nodes behind this gateway | +| `vpn_local_subnet` / `vpn_remote_subnet` | — | IPsec traffic selectors | +| `vpn_remote_gw_ip` | — | peer public IP; the fallback when `vpn_remote_addrs` is empty | +| `vpn_remote_addrs` | `[]` | peer outer IPs, one per tunnel | +| `vpn_tunnel_count` | `0` (auto) | one tunnel per entry in `vpn_remote_addrs`, minimum 1 | +| `vpn_id_style` | `""` (auto) | `fixed`, `indexed`, or `address` | +| `vpn_crypto_profile` | `gcmaes256` | `default` is the legacy AES-CBC profile | +| `vpn_bgp_enabled` | `false` | required for multi-gateway ECMP against a managed cloud peer | +| `vpn_xfrm_mtu` / `vpn_gre_mtu` | `1400` / `1400` | the tunnel MTU managed cloud VPNs document | +| `vpn_mss_clamp` / `_value` | `true` / `""` (derived) | see troubleshooting | +| `vpn_fou_disable_gro` | `true` | **required** for `gre_fou`; see troubleshooting | +| `vpn_perf_tuning` | `true` | offloads, queues, RPS, IRQ pinning, performance governor | +| `vpn_disable_rfs` | `true` | **keep it** — RFS reorders one SA's packets into anti-replay drops; measured 176× more retransmits when on | +| `vpn_replay_window` | `1024` | ESP anti-replay; 0–4096, and 32768 stops the SA installing | +| `vpn_check_crypto_accel` | `true` | warns when the kernel has no VAES AES-GCM driver (+23%/tunnel) | +| `vpn_require_crypto_accel` | `false` | make that a hard failure instead | +| `vpn_nic_rings` | `0` (leave) | raising to 8192 measured **−9.1%** | +| `vpn_rx_usecs` | `0` (adaptive) | fixed coalescing; 64 measured +2.6%, 128 measured −2.0% | +| `vpn_disable_conntrack` | `true` | NOTRACK; required for multi-gateway ECMP (`vpn_stateful_forward_rule` is therefore `false`) | +| `vpn_kernel_upgrade` / `vpn_kernel_reboot` | `true` / `true` | install a VAES kernel; reboot only a host with no VPN yet | +| `vpn_kernel_reboot_live` | `false` | allow that reboot on a host already carrying tunnels | + +Client (`roles/vpn_client/defaults/main.yml`): + +| Variable | Default | Purpose | +|----------|---------|---------| +| `vpn_gateway_hosts` | `groups['vpn_gateways']` | gateways to ECMP across; set per site in a two-site deployment | +| `vpn_remote_subnets` | — | CIDRs to route through the VPN | +| `vpn_client_tcp_tuning` | `true` | buffers + BBR; removes a ~1 Gbps/flow cap | +| `vpn_client_healthcheck` | auto | probe gateways, rebuild ECMP | + +DaemonSet (`k8s/vpn-client.yaml`): `GATEWAY_IPS`, `TRANSPORT`, `REMOTE_CIDRS`, +`DISABLE_GRO`, `MULTIPATH_HASH_POLICY`, `TCP_TUNING`. + +## Known limitations + +- **One client CIDR per gateway.** `vpn_client_cidr` is a single prefix. + Clients across disjoint subnets need a covering supernet. +- **`gre_fou` cannot serve a large client CIDR.** One permanent neighbour entry + per host: a `/20` is 4094, a `/16` is refused by the role. +- **`gre_fou` requires GRO off on every uplink.** Host-wide setting, affecting + all traffic on that NIC. +- **`route` transport needs port security disabled**, and is **the one path not + validated live** — it renders and passes static checks, but port security was + enabled throughout testing. Treat the first `route` deployment as supervised. +- **BGP is not validated live** either; it is not needed for a + Crusoe-to-Crusoe pairing. All Azure figures quoted come from Microsoft's + published tables, not measurement. +- **SNAT and multi-gateway ECMP are mutually exclusive.** The role refuses the + combination. +- **Ansible reports `changed`** on the script-driven tasks every run. The + scripts are idempotent — a re-run against a live deployment causes no traffic + loss — but the tasks are not annotated to detect a no-op. +- **Failover is probe-based**, so clients notice a dead gateway within + `vpn_client_healthcheck_interval` (15 s), not instantly. +- **Redeploys can leave a duplicate IKE SA.** Cosmetic — traffic uses one set + of kernel states — but `swanctl --list-sas` shows more SAs than tunnels. + `systemctl restart strongswan` clears them. +- **No IPv6.** ## Operations ```bash -ansible-playbook -i inventory.ini site.yml # Deploy all -ansible-playbook -i inventory.ini site.yml --limit gw-src # One gateway -ansible-playbook -i inventory.ini site.yml --tags verify # Check only -ansible-playbook -i inventory.ini site.yml --tags teardown # Remove all - -kubectl apply -f k8s/vpn-client.yaml # Deploy K8s DaemonSet -kubectl rollout restart ds/vpn-client -n kube-system # Restart DaemonSet -kubectl logs -n kube-system ds/vpn-client # Check logs +ansible-playbook -i inventory.ini site.yml # deploy all +ansible-playbook -i inventory.ini site.yml --limit gw-1 # one gateway (UDP probe skipped: peer not in run) +ansible-playbook -i inventory.ini site.yml --tags verify # check only +ansible-playbook -i inventory.ini site.yml --tags teardown # remove all (preflight skipped on purpose) + +kubectl apply -f k8s/vpn-client.yaml +kubectl rollout restart ds/vpn-client -n kube-system +kubectl logs -n kube-system ds/vpn-client ``` ## Troubleshooting ```bash # --- Gateway --- -sudo swanctl --list-sas # IPsec SA status -sudo swanctl --initiate --child site-tunnel # Manually initiate -sudo journalctl -u strongswan -f # StrongSwan logs -ip tunnel show gre-vpn # GRE tunnel status -ip neigh show dev gre-vpn | head # Neighbor table -ip route show table 100 # Policy routes (xfrm0 → GRE) -ip route show table 200 # Mark routes (GRE → xfrm0) -ip rule show # Routing rules -sudo iptables -t mangle -L PREROUTING -v -n # Mark rules -sudo iptables -t nat -L POSTROUTING -v -n # SNAT rules +sudo swanctl --list-sas # SA status and negotiated algorithms +ip -br link show type xfrm # every tunnel's interface +ip route show # one nexthop per tunnel +sysctl net.ipv4.fib_multipath_hash_policy # MUST be 1 with ECMP +ethtool -k ens3 | grep generic-receive # MUST be off with gre_fou +sudo iptables -t mangle -S FORWARD | grep TCPMSS +sudo vtysh -c "show ip bgp summary" # if BGP is enabled +sudo /usr/local/sbin/vpn-network.sh up # re-apply links/routes/marks +sudo conntrack -C # should be ~0 for tunnel traffic # --- Client VM --- -ip tunnel show gre-vpn -ip route show dev gre-vpn -systemctl status vpn-client +ip route show # one nexthop per gateway +sysctl net.ipv4.fib_multipath_hash_policy net.ipv4.tcp_congestion_control +systemctl status vpn-client vpn-client-health.timer +sudo /usr/local/sbin/vpn-client-health.sh # re-probe now -# --- K8s DaemonSet --- +# --- K8s --- kubectl logs -n kube-system ds/vpn-client -kubectl exec -n kube-system ds/vpn-client -- ip tunnel show gre-vpn -kubectl exec -n kube-system ds/vpn-client -- ip route show dev gre-vpn ``` | Symptom | Check | |---------|-------| -| IKE timeout | Firewall allows UDP 500+4500 between gateway public IPs? | -| Tunnel up, no traffic | Routes through xfrm0? Mark rules in place? (`ip rule show`) | -| FOU packets not arriving | Firewall allows UDP 9473 within VPC? | -| Full tunnel: no internet | Remote gateway has internet access? SNAT rule in place? | -| Full tunnel: SSH lost | Use jump host: `ssh -J user@gw-rmt user@` | -| New VM can't connect | Added to inventory + re-ran playbook? | -| K8s node can't connect | DaemonSet running? `kubectl get ds -n kube-system` | -| strongswan-starter conflict | Ansible disables it; if manual, run `systemctl stop strongswan-starter` | +| IKE timeout | UDP 500+4500 open **both ways** between every gateway public IP and the peer? Between gateways you own the preflight play tests exactly this; against a cloud peer check both firewalls — a one-way block looks exactly like a broken config. | +| `NO_PROPOSAL_CHOSEN` | Both ends offering the same crypto? A managed cloud peer usually needs a custom IPsec/IKE policy set to match — most default to AES-CBC. | +| **Ping and UDP fine, TCP collapses to a few Mbit/s** | **GRO on the physical NIC.** It coalesces inbound FOU packets that then cannot be re-encapsulated, so they are dropped — measured **3.65 Mbps vs 4590 Mbps**. `ethtool -K gro off` on gateways **and** clients; the role does this automatically for `gre_fou`. Look for `UdpInErrors` climbing on the receiving gateway. | +| TCP stalls only for full-size packets | MSS clamped **above** the path MTU. `iptables --set-mss` raises as well as lowers. Leave `vpn_mss_clamp_value` empty so it is derived. | +| Tunnel up, no traffic | Routes through xfrm? Mark rules present? (`ip rule show`) | +| `route` mode: tunnel up, no return traffic | **Port security still enabled on the gateway vNIC.** The usual cause. | +| N tunnels but no more throughput | `fib_multipath_hash_policy` = 1? Does `vpn_remote_addrs` have more than one address? Are per-SA byte counters even? | +| Throughput per flow stuck near 1 Gbps | Client TCP buffers. `vpn_client_tcp_tuning` and `tcp_rmem` max ≥ 64 MB. | +| **Tunnel healthy, ping clean, TCP retransmitting hard** | **Anti-replay is discarding reordered packets.** `grep XfrmInStateSeqError /proc/net/xfrm_stat` — if it climbs, check `cat /sys/class/net//queues/rx-0/rps_flow_cnt`. RFS hands one SA's packets between CPUs so they arrive out of order. Measured 18,624 retransmits with RFS on versus 106 with it off. Keep `vpn_disable_rfs: true`. | +| Tunnel never installs after changing `vpn_replay_window` | Values the kernel refuses leave the SA uninstalled. 1024 works; 32768 does not. The role now validates 0–4096. | +| Per-tunnel throughput ~20% below expectation | Kernel has no VAES AES-GCM driver. `sed -n "/^name .*: rfc4106(gcm(aes))$/,+1p" /proc/crypto` — want a `*vaes*` driver. Needs kernel ≥ 6.11; see [AZURE-10G-GUIDE.md](AZURE-10G-GUIDE.md) §4.2 (host-side, applies to any peer). | +| Benchmark plateaus at 4–5 Gbps regardless of tuning | A single `iperf3` server process is single-threaded. Use one process per flow — the `bandwidth-test` solution does. | +| Some flows blackhole after a gateway dies | Health timer running? `vpn_client_healthcheck: true`? | +| Return traffic dropped intermittently | SNAT or a conntrack-only firewall rule in the path. Both break multi-gateway ECMP. | +| Client play fails on "could not resolve every gateway's private IP" | Ran with `--limit vpn_clients`, so the gateways play never set its facts. Run the whole `site.yml`, or set `vpn_local_gw_ip=` per gateway in the inventory. | +| FOU packets not arriving | UDP 9473 open within the VPC? | +| Full tunnel: no internet | Remote gateway has internet access and a SNAT rule? | +| `strongswan-starter` conflict | Ansible disables it; if manual, `systemctl stop strongswan-starter` | diff --git a/strongswan-ipsec/ansible/group_vars/all.yml b/strongswan-ipsec/ansible/group_vars/all.yml new file mode 100644 index 0000000..6193fbd --- /dev/null +++ b/strongswan-ipsec/ansible/group_vars/all.yml @@ -0,0 +1,124 @@ +--- +# ============================================================================= +# THE ONLY FILE YOU EDIT (besides inventory.ini) +# +# Five values. Everything else in this solution is already set correctly for a +# managed cloud VPN peer: GCMAES256, one tunnel per peer address, GRE-over-FOU +# to your VMs, a stateless datapath, every measured host tweak, and a +# VAES-capable kernel installed and booted if the running one predates 6.11. +# +# Then: ansible-playbook -i inventory.ini site.yml +# +# --------------------------------------------------------------------------- +# IF YOU KEEP YOUR INVENTORY SOMEWHERE ELSE, READ THIS +# --------------------------------------------------------------------------- +# This file sits next to site.yml, and Ansible ranks a PLAYBOOK-adjacent +# group_vars/all ABOVE an INVENTORY-adjacent one. So if you build your own +# inventory tree with its own group_vars/all.yml, the placeholders below win +# and the deployment refuses to start. +# +# Three ways out, any is fine: +# * edit THIS file (the intended flow - nothing else to think about) +# * put your values in an inventory group_vars/.yml instead of +# all.yml; a group file outranks a playbook-adjacent all.yml +# * pass them last: ansible-playbook ... -e @my-vars.yml +# ============================================================================= + +# --- 1. The shared secret ------------------------------------------------ +# Generate: openssl rand -base64 48 +# This exact string also goes into the Azure Connection's "Shared key". +# +# ENCRYPT IT BEFORE COMMITTING: +# ansible-vault encrypt_string --stdin-name vpn_psk +# and paste the result here in place of the plain string. +vpn_psk: "" + +# --- 2. The peer's public IPs ------------------------------------------- +# LIST BOTH addresses of an Azure gateway in active-active mode. You get one +# tunnel per address automatically, and two tunnels is the most Azure allows +# a single on-premises device. +# +# Azure portal: Virtual Network Gateway > Properties > the two public IPs. +# Or: az network vnet-gateway show -g -n \ +# --query "bgpSettings.bgpPeeringAddresses[].tunnelIpAddresses" -o tsv +# +# One address is fine too - you simply get one tunnel. +vpn_remote_addrs: + - "" + - "" + +# --- 3. Your Crusoe subnet ---------------------------------------------- +# The CIDR your Crusoe VMs live in. Announced to the peer as the local +# traffic selector. +vpn_local_subnet: "" + +# --- 4. The peer's subnet ----------------------------------------------- +# The CIDR on the far side - your Azure VNet address space. Your client VMs +# get a route to this through the VPN. +# +# For a full tunnel (send ALL internet traffic through the peer) use +# "0.0.0.0/0" here and read the "Full Tunnel Mode" section of the README +# first: the peer needs a SNAT rule and internet access. +vpn_remote_subnet: "" + +# --- 5. Which Crusoe IPs join the overlay ------------------------------- +# Every client VM listed in inventory.ini must fall inside this CIDR. It sizes +# the GRE neighbour table, the mark rules and the firewall. +# +# Usually identical to vpn_local_subnet. Keep it no larger than a /20: the +# overlay pre-populates one permanent neighbour entry per address, and the +# role refuses a CIDR big enough to overflow the kernel's neighbour table. +vpn_client_cidr: "" + + +# ============================================================================= +# Below here: only if the defaults do not suit you. +# ============================================================================= + +# --- More than one gateway VM ------------------------------------------- +# Add gateway VMs to inventory.ini and they are used automatically - clients +# ECMP across every host in the vpn_gateways group with no list to maintain. +# +# AZURE REQUIRES BGP FOR THIS. Azure will not spread return traffic across +# several on-premises devices without it: it wants one Local Network Gateway +# per device, each with a unique public IP, ASN and BGP peer address. +# Two Crusoe sites you own need no BGP at all - that was validated live with +# up to five gateways per side using static routes. +# +# NOT VALIDATED against a real Azure peer. See the README's limitations. +# vpn_bgp_enabled: true +# vpn_bgp_peer_asn: 65515 # Azure's default +# vpn_bgp_advertise: [""] # same on every gateway +# vpn_bgp_accept: [""] +# (vpn_bgp_local_asn and vpn_bgp_peer_addrs are per gateway - see host_vars) + +# --- Dropping GRE --------------------------------------------------------- +# "route" removes the GRE overlay: no GRO constraint, and no per-host +# neighbour entry, so client CIDRs larger than a /20 become possible. +# +# PREREQUISITE NO PLAYBOOK CAN MEET: Crusoe NetEng must DISABLE PORT SECURITY +# on every gateway VM's vNIC. Without it the decrypted return traffic is +# dropped by the fabric and the tunnel looks healthy while nothing arrives. +# That manual step is why gre_fou is the default. +# vpn_client_transport: "route" + +# --- Kernel upgrade ------------------------------------------------------- +# ON by default. It installs linux-image-generic-6.11 and reboots, but only +# when the CPU advertises "vaes" AND the running kernel is not already using +# a VAES AES-GCM driver. Worth ~23% per tunnel (measured 4.67 -> 5.75 Gbps). +# On a first deploy it fires once, before any VPN config exists. On a host that +# already carries tunnels it installs the kernel and waits for you to reboot - +# it never drops live traffic on its own, and never reboots a host that is +# already running the requested kernel. +# +# vpn_kernel_upgrade: false # skip it entirely +# vpn_kernel_reboot: false # install it, reboot yourself later +# vpn_kernel_reboot_live: true # allow the reboot on a gateway with live tunnels +# vpn_kernel_package: "linux-image-generic-6.14" +# +# Do NOT point this at 7.0. Measured on the same hardware it kept the fast +# crypto and was still 2.4x slower end to end. + +# --- Connection settings -------------------------------------------------- +# Uncomment and set if your SSH user or key differ from the inventory's. +# ansible_user: ubuntu diff --git a/strongswan-ipsec/ansible/group_vars/remote_clients.yml b/strongswan-ipsec/ansible/group_vars/remote_clients.yml deleted file mode 100644 index 94de1ae..0000000 --- a/strongswan-ipsec/ansible/group_vars/remote_clients.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Remote-side client VMs (Crusoe-to-Crusoe only) -# -# These VMs sit behind gw-rmt. The vpn_client role creates a GRE-over-FOU -# tunnel from each VM to the gateway specified by vpn_gateway_host. -# Skip this file if the remote side is Azure/GCP/AWS (no VM config needed). - -# Which gateway to connect to -vpn_gateway_host: "gw-rmt" - -# CIDRs to route through the GRE tunnel to the gateway. -# This should be the Crusoe source subnet — traffic to these IPs goes -# through the tunnel to reach the source side. -vpn_remote_subnet: "" diff --git a/strongswan-ipsec/ansible/group_vars/source_clients.yml b/strongswan-ipsec/ansible/group_vars/source_clients.yml deleted file mode 100644 index 9aad301..0000000 --- a/strongswan-ipsec/ansible/group_vars/source_clients.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Source-side Crusoe client VMs -# -# These VMs sit behind gw-src. The vpn_client role creates a GRE-over-FOU -# tunnel from each VM to the gateway specified by vpn_gateway_host. - -# Which gateway to connect to (must match a host in [vpn_gateways]) -vpn_gateway_host: "gw-src" - -# CIDRs to route through the GRE tunnel to the gateway. -# For split tunnel (datacenter only): -# vpn_remote_subnet: "" -# For full tunnel (all internet via remote gateway): -vpn_remote_subnets: - - "0.0.0.0/1" - - "128.0.0.0/1" diff --git a/strongswan-ipsec/ansible/group_vars/vpn_gateways.yml b/strongswan-ipsec/ansible/group_vars/vpn_gateways.yml deleted file mode 100644 index 3b38a98..0000000 --- a/strongswan-ipsec/ansible/group_vars/vpn_gateways.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Shared settings for all VPN gateways. -# -# The PSK must be identical on both gateways (and must match the K8s -# DaemonSet Secret if using k8s/vpn-client.yaml). - -# IPsec pre-shared key (generate with: openssl rand -base64 48) -vpn_psk: "" diff --git a/strongswan-ipsec/ansible/host_vars/gw-2.yml.example b/strongswan-ipsec/ansible/host_vars/gw-2.yml.example new file mode 100644 index 0000000..4ded94c --- /dev/null +++ b/strongswan-ipsec/ansible/host_vars/gw-2.yml.example @@ -0,0 +1,58 @@ +# ============================================================================= +# OPTIONAL - and inert. Ansible ignores ".example" files. +# +# You need nothing in host_vars for the normal case: one Crusoe gateway +# talking to a managed cloud peer. group_vars/all.yml covers it. +# +# This file exists for the two cases where per-gateway values differ. +# Copy it to host_vars/.yml to use it. +# ============================================================================= + +# --------------------------------------------------------------------------- +# CASE 1: Crusoe to Crusoe (you own both sides) +# --------------------------------------------------------------------------- +# Both sides run this role, so the subnets must MIRROR each other and cannot +# come from a single shared file. Put the site-local values here, per gateway. +# +# On the site-A gateway: +# vpn_local_subnet: "10.0.0.0/20" # A +# vpn_remote_subnet: "10.1.0.0/20" # B +# vpn_client_cidr: "10.0.0.0/20" # A +# vpn_remote_gw_ip: "" +# +# On the site-B gateway, swap local and remote. +# +# Pair the gateways 1:1 by index (site-a-gw-N <-> site-b-gw-N) and let the +# clients ECMP across them. No BGP needed between sites you own - that was +# validated live with five gateways per side. +# +# IMPORTANT for two-site deployments: the clients' vpn_gateway_hosts defaults +# to EVERY host in the vpn_gateways group, which is right when all your +# gateways face the same peer, and wrong here - site-A clients must not route +# through site-B's gateways. Split the inventory into per-site groups and set +# the list explicitly, e.g. in group_vars/site_a_clients.yml: +# +# vpn_gateway_hosts: ["site-a-gw-1", "site-a-gw-2"] +# vpn_remote_subnets: ["10.1.0.0/20"] # site B + +# --------------------------------------------------------------------------- +# CASE 2: several gateway VMs against Azure, which requires BGP +# --------------------------------------------------------------------------- +# Azure wants one Local Network Gateway per on-premises device, each with a +# unique ASN and BGP peer address. Those two values are the only things that +# differ per gateway - vpn_bgp_advertise and vpn_bgp_accept stay identical +# everywhere and live in group_vars/all.yml. +# +# vpn_bgp_local_asn: 65012 # 65011 on gw-1, 65012 on gw-2, ... +# vpn_bgp_peer_addrs: # one per tunnel, from the gateway's +# - "" # bgpSettings.bgpPeeringAddresses +# - "" +# +# NOT VALIDATED against a real Azure peer. See the README's limitations. + +# --------------------------------------------------------------------------- +# Also per-gateway, rarely needed +# --------------------------------------------------------------------------- +# Lets you configure clients on their own with "--limit vpn_clients". Normally +# the gateway play discovers this and the client play reads it from facts. +# vpn_local_gw_ip: "" diff --git a/strongswan-ipsec/ansible/host_vars/gw-rmt.yml b/strongswan-ipsec/ansible/host_vars/gw-rmt.yml deleted file mode 100644 index a7df1d4..0000000 --- a/strongswan-ipsec/ansible/host_vars/gw-rmt.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -# Remote Gateway -# -# This is the VPN gateway on the remote side (datacenter, another Crusoe -# region, or a public cloud). It terminates IPsec from the source gateway. -# -# For non-Crusoe (Azure/GCP/AWS/datacenter): set vpn_use_gre=false -# Traffic is forwarded directly onto the local network (IP forwarding). -# For Crusoe-to-Crusoe: set vpn_use_gre=true + vpn_client_cidr -# GRE-over-FOU overlay to deliver traffic to local VMs. - -# IKE identity for this gateway (local) and the remote peer -vpn_local_id: "gw-rmt@crusoe.vpn" -vpn_remote_id: "gw-src@crusoe.vpn" - -# Public IP of the source gateway (IPsec peer) -vpn_remote_gw_ip: "" - -# IPsec traffic selectors — must mirror the source gateway. -# local = CIDRs on this side (0.0.0.0/0 = accept all traffic for full tunnel) -# remote = CIDRs on the source side (Crusoe subnets) -vpn_local_subnet: "0.0.0.0/0" -vpn_remote_subnet: "" - -# GRE-over-FOU overlay — set to true when this gateway is also on Crusoe. -# vpn_client_cidr: subnet of VMs behind this gateway -# For non-Crusoe: set vpn_use_gre=false (no vpn_client_cidr needed) -vpn_use_gre: false -# vpn_use_gre: true -# vpn_client_cidr: "" diff --git a/strongswan-ipsec/ansible/host_vars/gw-src.yml b/strongswan-ipsec/ansible/host_vars/gw-src.yml deleted file mode 100644 index 30ceb40..0000000 --- a/strongswan-ipsec/ansible/host_vars/gw-src.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# Source Gateway — Crusoe Cloud -# -# This is the VPN gateway on the Crusoe side. It terminates IPsec to the -# remote gateway and runs a GRE-over-FOU overlay to deliver traffic to -# local VMs and K8s nodes. - -# IKE identity for this gateway (local) and the remote peer -vpn_local_id: "gw-src@crusoe.vpn" -vpn_remote_id: "gw-rmt@crusoe.vpn" - -# Public IP of the remote gateway (IPsec peer) -vpn_remote_gw_ip: "" - -# IPsec traffic selectors — define what traffic enters the tunnel. -# local = CIDRs on this side (Crusoe subnets) -# remote = CIDRs on the other side (0.0.0.0/0 = full tunnel, all traffic) -vpn_local_subnet: "" -vpn_remote_subnet: "0.0.0.0/0" - -# GRE-over-FOU overlay — needed on Crusoe because port_security blocks -# direct forwarding of packets with non-matching source IPs. -# vpn_use_gre: enables the multipoint GRE tunnel on this gateway -# vpn_client_cidr: subnet of VMs/K8s nodes behind this gateway -# (used to pre-populate the GRE neighbor table) -vpn_use_gre: true -vpn_client_cidr: "" diff --git a/strongswan-ipsec/ansible/inventory.ini b/strongswan-ipsec/ansible/inventory.ini index 45bc107..c4c6cd2 100644 --- a/strongswan-ipsec/ansible/inventory.ini +++ b/strongswan-ipsec/ansible/inventory.ini @@ -1,41 +1,36 @@ -# =========================================================================== -# VPN Inventory +# ============================================================================= +# WHO to configure. The other file you edit is group_vars/all.yml (WHAT). # -# Gateways + Crusoe client VMs. Remote cloud VMs (Azure/GCP/AWS) are NOT -# listed — their cloud route tables handle routing automatically. -# =========================================================================== +# Only Crusoe machines go here. Remote cloud VMs (Azure/GCP/AWS) need nothing +# installed - their cloud route tables already point at the VPN gateway. +# ============================================================================= -# --- Gateways --- -[source_gateway] -gw-src ansible_host= - -# Comment out if strongSwan is already set up on the remote end (e.g., -# customer-managed Azure/GCP/AWS VPN gateway). -[remote_gateway] -gw-rmt ansible_host= - -# --- Crusoe client VMs (source side) --- -[source_clients] -vm-s1 ansible_host= -# vm-s2 ansible_host= - -# --- Crusoe client VMs (remote side, required if setting up Crusoe↔Crusoe VPN) --- -[remote_clients] -# vm-r1 ansible_host= -# vm-r2 ansible_host= - -# --- Aggregate groups --- -[vpn_gateways:children] -source_gateway -remote_gateway +# --- Your gateway VM(s) --- +# START WITH ONE. It holds one tunnel per address in vpn_remote_addrs, so an +# Azure active-active peer gives you two tunnels from this single VM - which +# is the most Azure allows one on-premises device. +# +# 8 vCPU is the right size. Each tunnel's inbound decrypt pins one core and +# cannot be spread, so a bigger instance does not go faster. +# +# Need more than ~4.6 Gbps through Azure? Add gateway VMs here - they are +# picked up automatically, no list to maintain. Azure then REQUIRES BGP; see +# group_vars/all.yml. +[vpn_gateways] +gw-1 ansible_host= +# gw-2 ansible_host= -[vpn_clients:children] -source_clients -remote_clients +# --- The Crusoe VMs that need to reach the remote subnet --- +# Every one of these must fall inside vpn_client_cidr. +# Using Kubernetes instead? Skip this group and deploy k8s/vpn-client.yaml. +[vpn_clients] +vm-1 ansible_host= +# vm-2 ansible_host= +# vm-3 ansible_host= # --- Connection defaults --- [all:vars] ansible_user=ubuntu ansible_become=true ansible_ssh_private_key_file=~/.ssh/ -ansible_ssh_extra_args=-o StrictHostKeyChecking=no +ansible_ssh_common_args=-o StrictHostKeyChecking=no diff --git a/strongswan-ipsec/ansible/roles/vpn_client/defaults/main.yml b/strongswan-ipsec/ansible/roles/vpn_client/defaults/main.yml index 87c5091..035541f 100644 --- a/strongswan-ipsec/ansible/roles/vpn_client/defaults/main.yml +++ b/strongswan-ipsec/ansible/roles/vpn_client/defaults/main.yml @@ -1,4 +1,115 @@ --- -vpn_gre_mtu: 1360 +# =========================================================================== +# vpn_client role defaults +# +# Configures a Crusoe VM to reach the remote site through one or more VPN +# gateways. With one gateway and no new variables set, this behaves exactly +# as the single-gateway version always did. +# +# Design: README.md, "Multiple tunnels and multiple gateways" +# =========================================================================== + +# --------------------------------------------------------------------------- +# Which gateways to use +# --------------------------------------------------------------------------- +# List of gateway inventory hostnames. Traffic to the remote subnets is +# ECMP'd across all of them, hashed per flow. +# +# The single-gateway spelling vpn_gateway_host still works and is treated as +# a one-element list. +# +# Each gateway's private IP comes from its own vpn_local_gw_ip, which the +# vpn_gateway role sets as a fact. That means the gateways play must run +# first - which site.yml does. To configure clients on their own (e.g. +# "--limit vpn_clients"), set it explicitly per gateway in the inventory: +# gw-1 ansible_host= vpn_local_gw_ip= +# +# DEFAULT: every host in the inventory's vpn_gateways group. Adding a gateway +# to the inventory is therefore the only step needed to ECMP across it - the +# customer never maintains this list by hand. +# +# That default assumes every gateway faces the SAME peer, which is the normal +# case (one Crusoe site talking to Azure/GCP/AWS). In a Crusoe-to-Crusoe +# deployment it is wrong - site-A clients must not route through site-B's +# gateways - so split the inventory per site and set this explicitly there. +vpn_gateway_hosts: "{{ groups['vpn_gateways'] | default([]) }}" + +# vpn_remote_subnets is NOT defaulted here on purpose: tasks/main.yml falls +# back to [vpn_remote_subnet], the same value the gateways already need, so +# the customer states the remote CIDR exactly once. + +# --------------------------------------------------------------------------- +# Transport - must match the gateway +# --------------------------------------------------------------------------- +# gre_fou GRE-over-FOU overlay (default) +# route plain static routes via the gateway's private IP. Requires +# Crusoe port security to be DISABLED on the GATEWAY's vNIC. +# "" = inherit from the first gateway's own configuration, which is the safe +# default because a mismatch silently breaks the datapath. +vpn_client_transport: "" + +# GRE-over-FOU - must match the gateway +# Must match the gateway. See the vpn_gateway role for the MTU maths: the +# binding constraint is vpn_xfrm_mtu (1400), not the 1500-byte uplink. +vpn_gre_mtu: 1400 vpn_gre_key: 100 vpn_fou_port: 9473 +vpn_fou_sport_auto: false +vpn_fou_port_count: 1 + +# route transport: MTU applied to the ECMP route so TCP shrinks without +# relying on PMTUD. Azure's documented tunnel MTU is 1400. +vpn_client_route_mtu: 1400 + +# --------------------------------------------------------------------------- +# Health - clients do not run BGP, so they probe instead +# --------------------------------------------------------------------------- +# Probe each gateway and rebuild the ECMP route from the ones that answer. +# The gateway stops answering ICMP when it has no established SA, so this +# catches a live VM with a dead tunnel as well as a dead VM. +vpn_client_healthcheck: "{{ (vpn_gateway_hosts | length) > 1 }}" +vpn_client_healthcheck_interval: "15s" +vpn_client_probe_timeout: 1 + +# --------------------------------------------------------------------------- +# Performance - see README.md and AZURE-10G-GUIDE.md +# --------------------------------------------------------------------------- +# ECMP across the gateways. 0 (kernel default) hashes src/dst IP only, so +# every flow between the same pair of hosts takes ONE gateway and the rest +# sit idle. The single most missable line on the client side. +vpn_multipath_hash_policy: 1 + +# route transport: skip a nexthop whose neighbour entry is unreachable. Free +# dead-gateway detection, no agent involved. +vpn_multipath_use_neigh: 1 + +# With ECMP the reply may arrive from a different gateway than the request +# left through, which strict reverse-path filtering rejects. +vpn_rp_filter: 2 + +# --- long-fat-pipe TCP. The biggest client-side win. --- +# Bandwidth-delay product decides single-flow throughput. At 50 ms RTT, +# 10 Gbps needs ~62 MB in flight, but the distro default caps the receive +# buffer at 6 MB - roughly 1 Gbps per flow, no matter how many tunnels +# exist underneath. ON by default because it is purely corrective. +vpn_client_tcp_tuning: true +vpn_tcp_rmem: "4096 131072 134217728" +vpn_tcp_wmem: "4096 65536 134217728" +vpn_rmem_max: 268435456 +vpn_wmem_max: 268435456 +# CUBIC collapses its window on a public-internet path with even light loss +# and never recovers the pipe. BBR does not. Needs fq for pacing. +vpn_congestion_control: "bbr" +vpn_default_qdisc: "fq" + +# --- NIC queues and offloads. This is where extra queues actually pay off, +# --- because client traffic is real application flows. +vpn_client_perf_tuning: true +vpn_client_rx_queues: "" # "" = one combined queue per vCPU +# See vpn_fou_disable_gro in the vpn_gateway role: GRO on the physical NIC +# breaks GRE-over-FOU for TCP (measured 3.65 Mbps vs 4590 Mbps). Clients +# receive FOU too, so they need it off as well. +vpn_fou_disable_gro: true +vpn_client_nic_offloads: "gso on tso on" +vpn_client_netdev_budget: 600 +vpn_client_netdev_budget_usecs: 8000 diff --git a/strongswan-ipsec/ansible/roles/vpn_client/tasks/main.yml b/strongswan-ipsec/ansible/roles/vpn_client/tasks/main.yml index 70db0ff..44dd01e 100644 --- a/strongswan-ipsec/ansible/roles/vpn_client/tasks/main.yml +++ b/strongswan-ipsec/ansible/roles/vpn_client/tasks/main.yml @@ -1,71 +1,188 @@ --- # ============================================================================= -# vpn_client role — GRE-over-FOU tunnel to the local Crusoe gateway +# vpn_client role +# +# Points a Crusoe VM at one or more VPN gateways. Traffic to the remote CIDRs +# is ECMP'd across every gateway, hashed per flow by the kernel. +# +# Nothing in this path is stateful - no NAT, no conntrack - which is what lets +# a reply come back through a DIFFERENT gateway than the request left through. +# See README.md, "Rules that come with ECMP". # ============================================================================= -- name: Resolve gateway private IP from inventory +# --------------------------------------------------------------------------- +# Resolve gateways and transport +# --------------------------------------------------------------------------- + +# vpn_gateway_host (singular) is the old spelling and is treated as a +# one-element list, so existing group_vars keep working unchanged. +- name: Resolve the gateway list ansible.builtin.set_fact: - vpn_gateway_ip: "{{ hostvars[vpn_gateway_host]['vpn_local_gw_ip'] | default(hostvars[vpn_gateway_host]['ansible_default_ipv4']['address']) }}" - tags: [setup, teardown] + vpn_gateway_host_list: >- + {{ vpn_gateway_hosts if (vpn_gateway_hosts | length) > 0 + else ([vpn_gateway_host] if (vpn_gateway_host | default('')) else []) }} + tags: [setup, teardown, verify] + +- name: Require at least one gateway + ansible.builtin.assert: + that: (vpn_gateway_host_list | length) > 0 + fail_msg: >- + Set vpn_gateway_hosts (a list of gateway inventory hostnames) or the + older vpn_gateway_host (a single hostname). + tags: [setup, verify] -- name: Detect local private IP +- name: Detect the local private IP ansible.builtin.set_fact: vpn_local_ip: "{{ ansible_default_ipv4.address }}" - tags: [setup] + tags: [setup, verify] + +# Transport must match the gateway, and a mismatch breaks the datapath +# silently, so it is inherited from the gateway unless overridden here. +- name: Resolve the transport + vars: + _gw: "{{ vpn_gateway_host_list[0] }}" + _from_gateway: >- + {{ hostvars[_gw].vpn_transport + | default(hostvars[_gw].vpn_client_transport | default('', true), true) }} + ansible.builtin.set_fact: + vpn_transport: >- + {{ vpn_client_transport if vpn_client_transport + else (_from_gateway if _from_gateway + else ('gre_fou' + if (hostvars[_gw].vpn_use_gre | default(true) | bool) + else 'route')) }} + tags: [setup, teardown, verify] + +- name: Resolve the remote CIDRs + ansible.builtin.set_fact: + vpn_client_remote_cidrs: >- + {{ vpn_remote_subnets | default([vpn_remote_subnet]) }} + vpn_client_fou_ports: >- + {{ range(vpn_fou_port | int, (vpn_fou_port | int) + (vpn_fou_port_count | int)) | list }} + tags: [setup, teardown, verify] + +- name: Build the client path list + ansible.builtin.set_fact: + vpn_client_paths: "{{ lookup('template', 'client-paths.json.j2') | from_json }}" + tags: [setup, teardown, verify] -# ---------- FOU + GRE tunnel ---------- +- name: Validate configuration + ansible.builtin.assert: + that: + - vpn_transport in ['gre_fou', 'route'] + - vpn_client_remote_cidrs | length > 0 + - vpn_client_paths | rejectattr('gateway_ip') | list | length == 0 + fail_msg: >- + Could not resolve every gateway's private IP. The client role reads it + from each gateway's vpn_local_gw_ip, which the vpn_gateway role sets as + a fact - so running the gateways play first (as site.yml does) is + enough. If you want to configure clients on their own, for example with + "--limit vpn_clients", set vpn_local_gw_ip explicitly on each gateway + host in the inventory or host_vars: + "gw-1 ansible_host= vpn_local_gw_ip=". + Also check vpn_client_transport is gre_fou or route and that + vpn_remote_subnets is set. + tags: [setup, verify] + +# --------------------------------------------------------------------------- +# Kernel parameters +# --------------------------------------------------------------------------- -- name: Load FOU kernel module +- name: Load the BBR module community.general.modprobe: - name: fou + name: tcp_bbr state: present + failed_when: false + when: vpn_client_tcp_tuning | bool and vpn_congestion_control == 'bbr' tags: [setup] -- name: Remove stale tunnel and FOU port - ansible.builtin.shell: | - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true - changed_when: true - failed_when: false +- name: Configure sysctl + ansible.builtin.template: + src: sysctl-vpn-client.conf.j2 + dest: /etc/sysctl.d/99-vpn-client.conf + mode: "0644" tags: [setup] -- name: Create FOU receive endpoint - ansible.builtin.command: ip fou add port {{ vpn_fou_port }} ipproto 47 - changed_when: true +# See the note in the vpn_gateway role: "sysctl --system" applies +# /etc/sysctl.conf last, so a drop-in can be silently overridden. +- name: Apply sysctl now + ansible.builtin.shell: | + sysctl --system >/dev/null + sysctl -p /etc/sysctl.d/99-vpn-client.conf >/dev/null + changed_when: false tags: [setup] -- name: Create GRE-over-FOU tunnel to gateway +- name: Read back the sysctls that ECMP depends on ansible.builtin.shell: | - ip link add name gre-vpn type gre \ - local {{ vpn_local_ip }} remote {{ vpn_gateway_ip }} key {{ vpn_gre_key }} ttl 255 \ - encap fou encap-sport {{ vpn_fou_port }} encap-dport {{ vpn_fou_port }} - ip link set gre-vpn mtu {{ vpn_gre_mtu }} - ip link set gre-vpn up - changed_when: true + echo "hash=$(sysctl -n net.ipv4.fib_multipath_hash_policy)" + echo "rpf=$(sysctl -n net.ipv4.conf.all.rp_filter)" + echo "cc=$(sysctl -n net.ipv4.tcp_congestion_control)" + register: vpn_client_sysctl_live + changed_when: false + tags: [setup, verify] + +- name: Assert the critical sysctls actually took effect + ansible.builtin.assert: + that: + - "'hash=' ~ (vpn_multipath_hash_policy | string) in vpn_client_sysctl_live.stdout" + - "'rpf=' ~ (vpn_rp_filter | string) in vpn_client_sysctl_live.stdout" + fail_msg: >- + A sysctl this design depends on did not take effect. Live values: + {{ vpn_client_sysctl_live.stdout_lines }}. Something applied later is + overriding /etc/sysctl.d/99-vpn-client.conf - check /etc/sysctl.conf, + which "sysctl --system" applies last. + tags: [setup, verify] + +- name: "WARNING: congestion control is not what was requested" + ansible.builtin.debug: + msg: >- + Requested {{ vpn_congestion_control }} but the live value is + {{ vpn_client_sysctl_live.stdout_lines | select('search', '^cc=') | first | default('?') }}. + Over a long internet path this costs real throughput - CUBIC gives up its + window on loss and does not recover it. + when: + - vpn_client_tcp_tuning | bool + - "'cc=' ~ vpn_congestion_control not in vpn_client_sysctl_live.stdout" + tags: [setup, verify] + +# --------------------------------------------------------------------------- +# Paths and routes - one rendered script, used by the play AND by systemd +# --------------------------------------------------------------------------- + +- name: Load the FOU kernel module + community.general.modprobe: + name: fou + state: present + when: vpn_transport == 'gre_fou' tags: [setup] -# ---------- Routing ---------- +- name: Install NIC tuning script + ansible.builtin.template: + src: client-perf-tuning.sh.j2 + dest: /usr/local/sbin/vpn-client-perf-tuning.sh + mode: "0755" + when: vpn_client_perf_tuning | bool + tags: [setup] -- name: Route remote subnet through FOU tunnel - ansible.builtin.command: ip route replace {{ item }} dev gre-vpn - loop: "{{ vpn_remote_subnets | default([vpn_remote_subnet]) }}" +- name: Run NIC tuning + ansible.builtin.command: /usr/local/sbin/vpn-client-perf-tuning.sh + register: vpn_client_tuning changed_when: true + when: vpn_client_perf_tuning | bool tags: [setup] -# ---------- Firewall ---------- - -- name: Allow FOU from gateway - ansible.builtin.iptables: - chain: INPUT - protocol: udp - destination_port: "{{ vpn_fou_port }}" - source: "{{ vpn_gateway_ip }}" - jump: ACCEPT - failed_when: false +- name: Install vpn-client script + ansible.builtin.template: + src: vpn-client.sh.j2 + dest: /usr/local/sbin/vpn-client.sh + mode: "0755" tags: [setup] -# ---------- Persistence ---------- +- name: Bring up the path(s) and ECMP route(s) + ansible.builtin.command: /usr/local/sbin/vpn-client.sh up + register: vpn_client_up + changed_when: true + tags: [setup] - name: Write vpn-client systemd unit ansible.builtin.template: @@ -81,42 +198,149 @@ daemon_reload: true tags: [setup] -# ---------- Verify ---------- +# --------------------------------------------------------------------------- +# Health +# --------------------------------------------------------------------------- + +- name: Install gateway health script + ansible.builtin.template: + src: vpn-client-health.sh.j2 + dest: /usr/local/sbin/vpn-client-health.sh + mode: "0755" + when: vpn_client_healthcheck | bool + tags: [setup] + +- name: Write gateway health unit and timer + ansible.builtin.template: + src: "vpn-client-health.{{ item }}.j2" + dest: "/etc/systemd/system/vpn-client-health.{{ item }}" + mode: "0644" + loop: [service, timer] + when: vpn_client_healthcheck | bool + tags: [setup] + +- name: Enable gateway health timer + ansible.builtin.systemd: + name: vpn-client-health.timer + state: started + enabled: true + daemon_reload: true + when: vpn_client_healthcheck | bool + tags: [setup] + +# --------------------------------------------------------------------------- +# Verify +# --------------------------------------------------------------------------- + +- name: Verify every GRE device is up + ansible.builtin.shell: ip link show {{ item.dev }} | grep -qE 'UP|UNKNOWN' + loop: "{{ vpn_client_paths }}" + loop_control: + label: "{{ item.dev }}" + changed_when: false + when: vpn_transport == 'gre_fou' + tags: [setup, verify] + +- name: Count nexthops on the first remote route + ansible.builtin.shell: >- + set -o pipefail; + ip route show {{ vpn_client_remote_cidrs[0] }} | grep -c nexthop || true + args: + executable: /bin/bash + register: vpn_client_nexthops + changed_when: false + tags: [setup, verify] + +- name: Verify GRO is off on the uplink (GRE-over-FOU correctness) + ansible.builtin.shell: >- + set -o pipefail; + ethtool -k $(ip -o -4 route show default | awk '{print $5; exit}') + | awk '/generic-receive-offload/{print $2}' + args: + executable: /bin/bash + register: vpn_client_gro_state + changed_when: false + failed_when: false + when: vpn_transport == 'gre_fou' and (vpn_fou_disable_gro | bool) + tags: [setup, verify] -- name: Verify FOU tunnel is up - ansible.builtin.shell: ip link show gre-vpn | grep -qE 'UP|UNKNOWN' +- name: "WARNING: GRO is still on and will destroy TCP" + ansible.builtin.debug: + msg: >- + GRO is still ON for this client's uplink. Inbound FOU packets get + coalesced and dropped; UDP and ICMP keep working while TCP collapses + (measured 3.65 Mbps vs 4590 Mbps). See the README troubleshooting table. + when: + - vpn_transport == 'gre_fou' + - vpn_fou_disable_gro | bool + - (vpn_client_gro_state.stdout | default('') | trim) == 'on' + tags: [setup, verify] + +- name: Verify the ECMP hash policy took + ansible.builtin.command: sysctl -n net.ipv4.fib_multipath_hash_policy + register: vpn_client_hash_policy changed_when: false tags: [setup, verify] +- name: "WARNING: ECMP hash policy is L3-only" + ansible.builtin.debug: + msg: >- + net.ipv4.fib_multipath_hash_policy is + {{ vpn_client_hash_policy.stdout | trim }}, not 1. Every flow from this + client will take ONE gateway and the others will sit idle. The setup will + look correct and perform like a single gateway. + when: + - (vpn_client_paths | length) > 1 + - vpn_client_hash_policy.stdout | trim != '1' + tags: [setup, verify] + - name: Report status ansible.builtin.debug: msg: >- - Client {{ inventory_hostname }} ({{ vpn_local_ip }}) ready: - FOU tunnel to gateway {{ vpn_gateway_ip }}, - remote subnets {{ vpn_remote_subnets | default([vpn_remote_subnet]) | join(', ') }} via gre-vpn + Client {{ inventory_hostname }} ({{ vpn_local_ip }}): + transport={{ vpn_transport }} + | {{ vpn_client_paths | length }} path(s) to + {{ vpn_client_paths | map(attribute='gateway_ip') | unique | join(', ') }} + | ECMP nexthops installed={{ vpn_client_nexthops.stdout | trim }} + | remote={{ vpn_client_remote_cidrs | join(', ') }} + | tcp_tuning={{ vpn_client_tcp_tuning | bool }} ({{ vpn_congestion_control }}) + | health={{ vpn_client_healthcheck | bool }} tags: [setup, verify] -# ---------- Teardown ---------- +# --------------------------------------------------------------------------- +# Teardown +# --------------------------------------------------------------------------- -- name: "Teardown: stop and disable vpn-client service" +- name: "Teardown: stop services" ansible.builtin.systemd: - name: vpn-client + name: "{{ item }}" state: stopped enabled: false + loop: [vpn-client-health.timer, vpn-client] failed_when: false tags: [teardown, never] -- name: "Teardown: remove systemd unit" +- name: "Teardown: tear down the paths" + ansible.builtin.command: /usr/local/sbin/vpn-client.sh down + failed_when: false + changed_when: true + tags: [teardown, never] + +- name: "Teardown: remove units, scripts and configs" ansible.builtin.file: - path: /etc/systemd/system/vpn-client.service + path: "{{ item }}" state: absent + loop: + - /etc/systemd/system/vpn-client.service + - /etc/systemd/system/vpn-client-health.service + - /etc/systemd/system/vpn-client-health.timer + - /usr/local/sbin/vpn-client.sh + - /usr/local/sbin/vpn-client-health.sh + - /usr/local/sbin/vpn-client-perf-tuning.sh + - /etc/sysctl.d/99-vpn-client.conf tags: [teardown, never] -- name: "Teardown: remove tunnel and FOU" - ansible.builtin.shell: | - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true - failed_when: false - changed_when: true +- name: "Teardown: reload systemd" + ansible.builtin.systemd: + daemon_reload: true tags: [teardown, never] diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/client-paths.json.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/client-paths.json.j2 new file mode 100644 index 0000000..0fe17a6 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/client-paths.json.j2 @@ -0,0 +1,39 @@ +{#- + One entry per client -> gateway path. With N gateways and M FOU ports there + are N*M paths, and every one becomes a nexthop of a single ECMP route. + + Gateway addresses are resolved straight from inventory here, so there is one + place that knows how to find them. + + Backward compatibility: a single path in gre_fou mode keeps the device name + "gre-vpn" exactly as this role has always used. +-#} +{%- set gws = [] -%} +{%- for h in vpn_gateway_host_list -%} +{%- set hv = hostvars[h] -%} +{#- vpn_local_gw_ip is the fact the vpn_gateway role sets; ansible_default_ipv4 + is the fallback for a gateway the gateway play has not visited yet. -#} +{%- if hv.vpn_local_gw_ip is defined and hv.vpn_local_gw_ip -%} +{%- set ip = hv.vpn_local_gw_ip -%} +{%- elif hv.ansible_default_ipv4 is defined -%} +{%- set ip = hv.ansible_default_ipv4.address -%} +{%- else -%} +{%- set ip = '' -%} +{%- endif -%} +{%- set _ = gws.append({'host': h, 'ip': ip}) -%} +{%- endfor -%} +{%- set paths = [] -%} +{%- set total = (gws | length) * (vpn_fou_port_count | int) -%} +{%- for gw in gws -%} +{%- for p in range(vpn_fou_port_count | int) -%} +{%- set n = paths | length -%} +{%- set _ = paths.append({ + 'index': n, + 'gateway_host': gw.host, + 'gateway_ip': gw.ip, + 'dev': 'gre-vpn' if total == 1 else 'gre-vpn' ~ n, + 'fou_port': (vpn_fou_port | int) + p, + 'gre_key': (vpn_gre_key | int) + p }) -%} +{%- endfor -%} +{%- endfor -%} +{{ paths | to_json }} diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/client-perf-tuning.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/client-perf-tuning.sh.j2 new file mode 100644 index 0000000..4dbe1e9 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/client-perf-tuning.sh.j2 @@ -0,0 +1,32 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# NIC queues and offloads on the client. Unlike the gateway, extra queues DO +# pay off here: client traffic is real application flows, so there is enough +# flow diversity for RSS to spread across them. +set -u + +UPLINK=$(ip -o -4 route show default | awk '{print $5; exit}') +NCPU=$(nproc) +{% if vpn_client_rx_queues %} +WANT={{ vpn_client_rx_queues }} +{% else %} +WANT="$NCPU" +{% endif %} + +log() { echo "client-perf-tuning: $*"; } +[ -n "${UPLINK:-}" ] || { log "no default route - nothing to tune"; exit 0; } + +MAXQ=$(ethtool -l "$UPLINK" 2>/dev/null | awk '/^Combined:/{print $2; exit}') +if [ -n "${MAXQ:-}" ] && [ "$MAXQ" -gt 0 ] 2>/dev/null; then + [ "$WANT" -gt "$MAXQ" ] && WANT="$MAXQ" + ethtool -L "$UPLINK" combined "$WANT" 2>/dev/null \ + && log "$UPLINK combined queues -> $WANT (max $MAXQ)" \ + || log "could not set combined queues on $UPLINK" +else + log "$UPLINK does not report combined queues" +fi + +ethtool -K "$UPLINK" {{ vpn_client_nic_offloads }} 2>/dev/null \ + && log "offloads: {{ vpn_client_nic_offloads }}" || log "some offloads unsupported" +exit 0 diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/sysctl-vpn-client.conf.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/sysctl-vpn-client.conf.j2 new file mode 100644 index 0000000..0bdfab8 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/sysctl-vpn-client.conf.j2 @@ -0,0 +1,40 @@ +# Managed by Ansible - vpn_client role +# +# --- ECMP across the VPN gateways --- +# 0 (kernel default) hashes source/destination IP only, so every flow between +# the same pair of hosts takes ONE gateway and the others sit idle. The setup +# looks correct and performs like a single gateway. +net.ipv4.fib_multipath_hash_policy = {{ vpn_multipath_hash_policy }} +{% if vpn_transport == 'route' %} + +# Skip a nexthop whose neighbour entry has gone unreachable. Free dead-gateway +# detection for route transport (a GRE device stays UP regardless, which is why +# gre_fou needs the probe instead). +net.ipv4.fib_multipath_use_neigh = {{ vpn_multipath_use_neigh }} +{% endif %} + +# The reply may arrive from a different gateway than the request left through. +net.ipv4.conf.all.rp_filter = {{ vpn_rp_filter }} +net.ipv4.conf.default.rp_filter = {{ vpn_rp_filter }} +{% if vpn_client_tcp_tuning | bool %} + +# --- long-fat-pipe TCP --- +# 10 Gbps at 50 ms RTT needs ~62 MB in flight. The 6 MB distro default caps a +# single flow near 1 Gbps regardless of how many tunnels are underneath. +net.core.rmem_max = {{ vpn_rmem_max }} +net.core.wmem_max = {{ vpn_wmem_max }} +net.ipv4.tcp_rmem = {{ vpn_tcp_rmem }} +net.ipv4.tcp_wmem = {{ vpn_tcp_wmem }} +net.ipv4.tcp_window_scaling = 1 + +# CUBIC gives up the window on a lossy internet path and does not get it back. +net.core.default_qdisc = {{ vpn_default_qdisc }} +net.ipv4.tcp_congestion_control = {{ vpn_congestion_control }} +{% endif %} +{% if vpn_client_perf_tuning | bool %} + +# --- softirq headroom --- +net.core.netdev_max_backlog = 250000 +net.core.netdev_budget = {{ vpn_client_netdev_budget }} +net.core.netdev_budget_usecs = {{ vpn_client_netdev_budget_usecs }} +{% endif %} diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.service.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.service.j2 new file mode 100644 index 0000000..303bd6d --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.service.j2 @@ -0,0 +1,8 @@ +# Managed by Ansible - {{ inventory_hostname }} +[Unit] +Description=VPN client gateway health (rebuild ECMP from reachable gateways) +After=vpn-client.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/vpn-client-health.sh diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.sh.j2 new file mode 100644 index 0000000..da0b310 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# Clients do not run BGP, so this is how they learn that a gateway is gone. +# Probe each gateway and rebuild the ECMP route from the ones that answer. +# +# The probe covers BOTH failure modes, because the gateway's own health timer +# stops answering ICMP when it has no established SA. So a live VM with a dead +# tunnel drops out too, not just a dead VM. +# +# Re-hashing is safe here: nothing in this datapath is stateful, so a flow that +# moves to a different gateway keeps working. That is the property NAT would +# have destroyed. +set -u + +REMOTE_CIDRS="{{ vpn_client_remote_cidrs | join(' ') }}" +STATE_FILE=/run/vpn-client-health.state +TIMEOUT={{ vpn_client_probe_timeout }} + +alive=() +alive_count=0 + +{% for p in vpn_client_paths %} +if ping -c 1 -W "$TIMEOUT" -n -q {{ p.gateway_ip }} >/dev/null 2>&1; then +{% if vpn_transport == 'route' %} + alive+=(nexthop via {{ p.gateway_ip }} weight 1) +{% else %} + alive+=(nexthop dev {{ p.dev }} weight 1) +{% endif %} + alive_count=$(( alive_count + 1 )) +fi +{% endfor %} + +if [ "$alive_count" -eq 0 ]; then + # Every gateway is unreachable. Leave the route as it is: a stale route has + # a chance of recovering, a deleted one is a guaranteed outage. + echo "vpn-client-health: no gateway answered - leaving routes alone" >&2 + exit 0 +fi + +desired="${alive[*]}" +previous=$(cat "$STATE_FILE" 2>/dev/null || true) +[ "$desired" = "$previous" ] && exit 0 + +for cidr in $REMOTE_CIDRS; do + # shellcheck disable=SC2086 + ip route replace "$cidr" "${alive[@]}" {{ 'mtu ' ~ vpn_client_route_mtu if (vpn_transport == 'route' and vpn_client_route_mtu) else '' }} +done +printf '%s' "$desired" > "$STATE_FILE" +echo "vpn-client-health: $alive_count/{{ vpn_client_paths | length }} gateway(s) up; routes rebuilt" diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.timer.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.timer.j2 new file mode 100644 index 0000000..51cbbc5 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client-health.timer.j2 @@ -0,0 +1,11 @@ +# Managed by Ansible - {{ inventory_hostname }} +[Unit] +Description=Run VPN client gateway health every {{ vpn_client_healthcheck_interval }} + +[Timer] +OnBootSec=30s +OnUnitActiveSec={{ vpn_client_healthcheck_interval }} +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.service.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.service.j2 index 51cd59b..380e2b1 100644 --- a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.service.j2 +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.service.j2 @@ -1,6 +1,6 @@ -# Managed by Ansible — {{ inventory_hostname }} +# Managed by Ansible - {{ inventory_hostname }} [Unit] -Description=VPN Client GRE-over-FOU Tunnel +Description=VPN Client path(s) to the VPN gateway(s) After=network-online.target Wants=network-online.target @@ -9,26 +9,8 @@ Type=oneshot RemainAfterExit=yes Restart=on-failure RestartSec=5 -ExecStart=/bin/bash -c '\ - modprobe fou; \ - LOCAL_IP=$(ip -4 route get 1.1.1.1 | awk "/src/{print \$$7; exit}"); \ - ip link del gre-vpn 2>/dev/null || true; \ - ip tunnel del gre-vpn 2>/dev/null || true; \ - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true; \ - ip fou add port {{ vpn_fou_port }} ipproto 47; \ - ip link add name gre-vpn type gre \ - local $LOCAL_IP remote {{ vpn_gateway_ip }} key {{ vpn_gre_key }} ttl 255 \ - encap fou encap-sport {{ vpn_fou_port }} encap-dport {{ vpn_fou_port }}; \ - ip link set gre-vpn mtu {{ vpn_gre_mtu }}; \ - ip link set gre-vpn up; \ -{% for subnet in vpn_remote_subnets | default([vpn_remote_subnet]) %} - ip route replace {{ subnet }} dev gre-vpn; \ -{% endfor %} - echo "VPN client tunnel ready ($LOCAL_IP -> {{ vpn_gateway_ip }})"' -ExecStop=/bin/bash -c '\ - ip link del gre-vpn 2>/dev/null || true; \ - ip tunnel del gre-vpn 2>/dev/null || true; \ - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true' +ExecStart=/usr/local/sbin/vpn-client.sh up +ExecStop=/usr/local/sbin/vpn-client.sh down [Install] WantedBy=multi-user.target diff --git a/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.sh.j2 new file mode 100644 index 0000000..090a340 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_client/templates/vpn-client.sh.j2 @@ -0,0 +1,130 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# Brings up this client's path(s) to the VPN gateway(s) and installs one ECMP +# route per remote CIDR across them. Called by the Ansible play AND by +# vpn-client.service, so live state and boot state cannot drift. +# +# Usage: vpn-client.sh {up|down} +set -u + +{#- ECMP nexthop spec: device list for gre_fou, via-IP list for route. #} +{%- if vpn_transport == 'route' -%} +{%- set NH = vpn_client_paths | map(attribute='gateway_ip') + | map('regex_replace', '^(.*)$', 'nexthop via \\1 weight 1') | join(' ') -%} +{%- if vpn_client_paths | length == 1 -%} +{%- set NH = 'via ' ~ vpn_client_paths[0].gateway_ip -%} +{%- endif -%} +{%- set MTU_ARG = 'mtu ' ~ vpn_client_route_mtu if vpn_client_route_mtu else '' -%} +{%- else -%} +{%- set NH = vpn_client_paths | map(attribute='dev') + | map('regex_replace', '^(.*)$', 'nexthop dev \\1 weight 1') | join(' ') -%} +{%- if vpn_client_paths | length == 1 -%} +{%- set NH = 'dev ' ~ vpn_client_paths[0].dev -%} +{%- endif -%} +{%- set MTU_ARG = '' -%} +{%- endif %} + +# transport = {{ vpn_transport }}, paths = {{ vpn_client_paths | length }} +{% for p in vpn_client_paths %} +# path {{ p.index }}: {{ p.gateway_host }} ({{ p.gateway_ip }}){% if vpn_transport == 'gre_fou' %} via {{ p.dev }} on UDP {{ p.fou_port }}{% endif %} + +{% endfor %} +REMOTE_CIDRS="{{ vpn_client_remote_cidrs | join(' ') }}" + +{% if vpn_transport == 'gre_fou' %} +up_tunnels() { + modprobe fou 2>/dev/null || true +{% if vpn_fou_disable_gro | bool %} + # GRO on the physical NIC breaks GRE-over-FOU for TCP - see the + # vpn_gateway role's defaults. Applied unconditionally. + UPLINK=$(ip -o -4 route show default | awk '{print $5; exit}') + [ -n "${UPLINK:-}" ] && ethtool -K "$UPLINK" gro off 2>/dev/null || true +{% endif %} + # The local address is read at runtime, not baked in, so the unit survives + # a DHCP change across reboots. + LOCAL_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}') + if [ -z "${LOCAL_IP:-}" ]; then + echo "ERROR: could not determine the local IP" >&2 + exit 1 + fi + # Purge EVERY gre-vpn* device, not just the ones we are about to create. + # Changing the gateway count renames the devices (gre-vpn <-> gre-vpn0,1..), + # and a leftover device with the same local/remote/key blocks the new one. + for _d in $(ip -o link show 2>/dev/null | awk -F': ' '{print $2}' \ + | sed 's/@.*//' | grep -E '^gre-vpn[0-9]*$'); do + ip link del "$_d" 2>/dev/null || true + ip tunnel del "$_d" 2>/dev/null || true + done +{% for port in vpn_client_fou_ports %} + ip fou del port {{ port }} 2>/dev/null || true +{% endfor %} + sleep 1 +{% for port in vpn_client_fou_ports %} + ip fou add port {{ port }} ipproto 47 +{% endfor %} +{% for p in vpn_client_paths %} + ip link add name {{ p.dev }} type gre \ + local "$LOCAL_IP" remote {{ p.gateway_ip }} key {{ p.gre_key }} ttl 255 \ + encap fou encap-sport {{ 'auto' if vpn_fou_sport_auto | bool else p.fou_port }} encap-dport {{ p.fou_port }} + ip link set {{ p.dev }} mtu {{ vpn_gre_mtu }} + ip link set {{ p.dev }} up +{% endfor %} +{% for port in vpn_client_fou_ports %} +{% for gw in vpn_client_paths | map(attribute='gateway_ip') | unique %} + iptables -C INPUT -p udp --dport {{ port }} -s {{ gw }} -j ACCEPT 2>/dev/null \ + || iptables -I INPUT 1 -p udp --dport {{ port }} -s {{ gw }} -j ACCEPT +{% endfor %} +{% endfor %} +} +{% endif %} + +# One ECMP route per remote CIDR. Which gateway a flow uses is decided by the +# kernel's 5-tuple hash, so a given flow is stable but different flows spread. +# Requires net.ipv4.fib_multipath_hash_policy = 1 - without it every flow +# takes the same nexthop. +up_routes() { + for cidr in $REMOTE_CIDRS; do + # shellcheck disable=SC2086 + ip route replace "$cidr" {{ NH }} {{ MTU_ARG }} + done +} + +do_up() { + # The health timer remembers the last gateway set it routed through and + # only rewrites the ECMP route when that set changes. The full route is + # about to be reinstalled, so forget the memo and let the first tick + # reconcile against reality. + rm -f /run/vpn-client-health.state 2>/dev/null || true +{% if vpn_transport == 'gre_fou' %} + up_tunnels +{% endif %} + up_routes + echo "vpn-client up: {{ vpn_client_paths | length }} path(s), transport {{ vpn_transport }}" +} + +do_down() { + for cidr in $REMOTE_CIDRS; do + ip route del "$cidr" 2>/dev/null || true + done +{% if vpn_transport == 'gre_fou' %} + # Purge EVERY gre-vpn* device, not just the ones we are about to create. + # Changing the gateway count renames the devices (gre-vpn <-> gre-vpn0,1..), + # and a leftover device with the same local/remote/key blocks the new one. + for _d in $(ip -o link show 2>/dev/null | awk -F': ' '{print $2}' \ + | sed 's/@.*//' | grep -E '^gre-vpn[0-9]*$'); do + ip link del "$_d" 2>/dev/null || true + ip tunnel del "$_d" 2>/dev/null || true + done +{% for port in vpn_client_fou_ports %} + ip fou del port {{ port }} 2>/dev/null || true +{% endfor %} +{% endif %} + echo "vpn-client down" +} + +case "${1:-up}" in + up) do_up ;; + down) do_down ;; + *) echo "usage: $0 {up|down}" >&2; exit 2 ;; +esac diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/defaults/main.yml b/strongswan-ipsec/ansible/roles/vpn_gateway/defaults/main.yml index 2210d46..1ef6687 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/defaults/main.yml +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/defaults/main.yml @@ -1,23 +1,529 @@ --- -vpn_xfrm_iface: "xfrm0" -vpn_xfrm_if_id: 100 -vpn_xfrm_mtu: 1400 +# =========================================================================== +# vpn_gateway role defaults +# +# DEFAULTS ARE TUNED FOR A MANAGED CLOUD PEER (Azure/GCP/AWS). +# +# Out of the box this role gives you: one gateway VM, one tunnel per remote +# outer address (so 2 against an Azure active-active gateway), GCMAES256, +# GRE-over-FOU to local clients, a stateless datapath, every measured host +# tweak applied, and a VAES-capable kernel installed and booted if the running +# one predates 6.11. +# +# The customer edits ONE file: group_vars/all.yml. Everything below is already +# correct for that case. +# +# Sizing, transports, ciphers, cloud-side prerequisites: README.md +# Reaching 10 Gbps with N+1 against Azure: AZURE-10G-GUIDE.md +# =========================================================================== -# GRE-over-FOU (GRE encapsulated in UDP — works through firewalls that -# only allow TCP/UDP and block raw GRE protocol 47) -vpn_gre_mtu: 1360 -vpn_gre_key: 100 -vpn_fou_port: 9473 +# --------------------------------------------------------------------------- +# XFRM interfaces +# --------------------------------------------------------------------------- +# Tunnel i (0-based) gets interface "{{ vpn_xfrm_iface_prefix }}{{ i }}" and +# if_id "{{ vpn_xfrm_if_id }} + i" -> xfrm0/100, xfrm1/101, xfrm2/102 ... +vpn_xfrm_iface_prefix: "xfrm" +vpn_xfrm_iface: "xfrm0" # tunnel 0 - kept for backward compatibility +vpn_xfrm_if_id: 100 # tunnel 0, and the base for the fan-out +vpn_xfrm_mtu: 1400 # Azure's documented tunnel MTU. Do not raise. + +# --------------------------------------------------------------------------- +# Tunnel fan-out (ECMP) +# --------------------------------------------------------------------------- +# SIZING, measured across two Crusoe regions with 20 client pairs: +# +# Inbound ESP decrypt for ONE tunnel is pinned to ONE core. One tunnel is one +# outer flow, one RX queue, one CPU - and nothing software-side splits it +# (RPS hashes per flow; a 16 vCPU gateway still sat at 91% of a single core). +# +# That core sustains about 2.1-2.2 Gbps, so: +# throughput per gateway PAIR ~ 2.1 Gbps +# NIC load per gateway ~ 2x that, because every packet crosses the +# uplink twice (in as FOU, out as ESP) +# +# Scale by ADDING GATEWAYS, not by growing them. Measured: 1 gateway/site +# 2.05 Gbps, 2 gateways/site 4.38 Gbps - linear. An 8 vCPU / 10 Gbps-NIC +# instance is comfortable; 16 vCPU bought nothing. +# +# Number of parallel IPsec SAs from THIS gateway to the remote site. +# +# N tunnels only spread load if the OUTER flow varies. With NAT-T both UDP +# ports are pinned to 4500, so the only entropy is the IP pair. Raising this +# without giving vpn_remote_addrs (or vpn_local_addrs) more than one entry +# produces a setup that looks right and performs like a single tunnel. The +# role warns when that happens. +# 0 = AUTO: one tunnel per entry in vpn_remote_addrs, minimum 1. An Azure +# gateway in active-active mode publishes two public IPs, so listing both +# gives two tunnels with no extra config - which is the most a single +# on-premises device gets. Set a number to override. +# +# Two tunnels to the SAME remote IP are pointless: same outer 4-tuple means +# same RX queue means same CPU. Tunnels only help when the outer addresses +# differ, which is why this counts addresses rather than defaulting to 2. +vpn_tunnel_count: 0 + +# Remote (outer) addresses, one per tunnel. Empty = [vpn_remote_gw_ip] for all. +# For Azure active-active, list both gateway instance public IPs. +vpn_remote_addrs: [] + +# Local (outer) addresses, one per tunnel. Empty = "%any" for every tunnel, +# which is correct on Crusoe (one IP per VM, and public IPs are 1:1 NAT'd). +# Entries are a bare IP, or a dict: +# addr bindable local address -> swanctl "local_addrs" +# public address the peer sees; used as the IKE id when vpn_id_style=address +# iface uplink for this tunnel's XFRM device and source routing +# gateway next hop for that uplink (non-primary NICs only) +vpn_local_addrs: [] + +# Escape hatch: supply the fully-formed tunnel list and skip all derivation. +vpn_tunnels_override: [] + +# Prune dead tunnels from the local ECMP route, and signal tunnel health to +# clients (clients do not run BGP, so this is how they learn). An XFRM +# interface stays UP whether or not its SA is established. +# "" = auto: on whenever this gateway has more than one tunnel OR serves +# clients. The second half matters: with N single-tunnel gateways sharing the +# ECMP load, this timer is the ONLY way a client learns that one of them has +# lost its tunnel - clients do not run BGP, they just probe. Without it a +# gateway with a dead tunnel keeps answering and keeps its 1/N share of traffic. +vpn_ecmp_healthcheck: "" +vpn_ecmp_healthcheck_interval: "15s" + +# Source-routing tables for multi-uplink deployments (tunnel i -> base + i). +vpn_srcrt_table_base: 300 + +# --------------------------------------------------------------------------- +# Crypto profiles +# --------------------------------------------------------------------------- +# In strongSwan "aes256gcm128" and "aes256gcm16" are the SAME thing: +# AES-256-GCM with a 128-bit (16-byte) ICV. That is what Azure calls +# GCMAES256. Today's ESP is already GCMAES256; it is IKE that is AES-CBC. +# +# Azure's own per-tunnel numbers (VpnGw4/VpnGw5): +# GCMAES256 2.3 Gbps / 220,000 pps +# AES256 + SHA256 700 Mbps / 66,000 pps +# So the profile is worth ~3.3x per tunnel on the Azure side. +vpn_crypto_profile: "gcmaes256" + +vpn_crypto_profiles: + # Exactly what this role has always shipped. + default: + ike: "aes256-sha384-ecp256-modp3072" + esp: "aes256gcm128-ecp256-modp3072" + # AEAD end to end. Matches an Azure custom IPsec/IKE policy of + # GCMAES256 / GCMAES256 with PFS group ECP384. + gcmaes256: + ike: "aes256gcm16-prfsha384-ecp384" + esp: "aes256gcm16-ecp384" + # Same AEAD, cheaper PRF and DH group - throughput-first deployments. + gcmaes256_fast: + ike: "aes256gcm16-prfsha256-ecp256" + esp: "aes256gcm16-ecp256" + # For peers that will not negotiate AEAD at all. + cbc_compat: + ike: "aes256-sha256-modp2048" + esp: "aes256-sha256-modp2048" + +# Set either explicitly to override the profile. +vpn_ike_proposals: "{{ vpn_crypto_profiles[vpn_crypto_profile].ike }}" +vpn_esp_proposals: "{{ vpn_crypto_profiles[vpn_crypto_profile].esp }}" -vpn_ike_proposals: "aes256-sha384-ecp256-modp3072" -vpn_esp_proposals: "aes256gcm128-ecp256-modp3072" vpn_ike_rekey_time: "14400s" vpn_child_rekey_time: "3600s" vpn_dpd_delay: "30s" -# Policy routing table for xfrm → GRE forwarding -vpn_rt_table: 100 +# Force UDP encapsulation (port 4500) even when no NAT is detected. +# UDP-encapsulate ESP unconditionally. Managed cloud peers sit behind NAT +# more often than not, and forcing encap avoids a class of intermittent +# "tunnel up, no traffic" failures. Also gives the outer flow a UDP 4500 +# 4-tuple, which is what the NIC hashes on. +vpn_force_encap: true + +# "" = auto. "start" brings every tunnel up at deploy time; "trap" waits for +# the first matching packet. +# +# INTENTIONAL DEFAULT CHANGE, measured: "trap" is lazy, so in any scale-out +# deployment the tunnels come up one at a time as traffic happens to hash to +# them - and the flows that trigger them are dropped while IKE runs. Observed +# live with 5 gateways per site: only gateway 1 had an SA; the other four sat +# idle with no tunnel at all until traffic reached them. That silently +# understates throughput and drops real user traffic. +# +# Auto now resolves to: +# start whenever this gateway serves clients (gre_fou or route), or has +# more than one tunnel - i.e. every case where laziness costs traffic +# trap only for a plain point-to-point gateway with no local clients +vpn_start_action: "" + +# IKE identity style. +# fixed vpn_local_id / vpn_remote_id verbatim (today) +# indexed site-b-gw-1@..., site-b-gw-2@... - for Crusoe<->Crusoe fan-out +# address the tunnel's outer IP - what Azure/AWS/GCP peers require +# "" = auto: "fixed" for one tunnel, "address" when the tunnels have distinct +# remote outer IPs (the Azure case), "indexed" otherwise. +vpn_id_style: "" + +# IKE identities. Defaulted so the customer never has to set them. +# +# Azure identifies an on-premises device by its PUBLIC IP, and a Crusoe VM +# cannot read its own public IP from its interface - the fabric maps it to the +# private NIC address. But the customer already typed it into inventory.ini as +# ansible_host, so that is the authoritative source. +vpn_local_outer_ip: "{{ ansible_host | default(vpn_local_gw_ip | default('%any')) }}" +vpn_local_id: "{{ vpn_local_outer_ip }}" + +# The peer's identity is its outer address: the first entry of vpn_remote_addrs +# if given, else vpn_remote_gw_ip. With "address" style each tunnel uses its +# own remote address instead of this single value. +vpn_remote_id: >- + {{ (vpn_remote_addrs | first) if (vpn_remote_addrs | default([]) | length) > 0 + else (vpn_remote_gw_ip | default('%any')) }} + +# ESP anti-replay window, rendered on each child SA. +# +# Multi-queue receive and WAN reordering both deliver valid packets late, and +# the strongSwan default window (32) drops them at high pps. The drops are +# invisible: the tunnel stays INSTALLED, ping is clean, and only TCP suffers. +# Watch for them with: +# cat /proc/net/xfrm_stat | grep XfrmInStateSeqError +# ip -s xfrm state | grep replay-window +# +# Measured: 1024 rejected 28,290 packets per 20 s at 4.8 Gbps on a 54 ms path +# (0.28% loss). Setting it to 0 disables anti-replay and removed the drops +# entirely - retransmits fell from 17,110 to 62 - but throughput did NOT +# improve, so the drops were costing correctness noise, not bandwidth. +# Fixing RFS (vpn_disable_rfs) removes most of the reordering at the source +# and is the better lever. +# +# Values above 4096 are untested here and 32768 is KNOWN BAD: the SA fails to +# install and the tunnel never comes up. Validated in tasks/main.yml. +# +# 0 disables anti-replay protection. That is a real, if modest, security +# reduction on a public-internet tunnel - an attacker who can capture ESP +# packets can replay them. Do not set it to 0 to chase throughput. +vpn_replay_window: 1024 + +# --------------------------------------------------------------------------- +# Client transport (how local Crusoe VMs / K8s nodes reach this gateway) +# --------------------------------------------------------------------------- +# gre_fou GRE-over-FOU overlay (== the deprecated vpn_use_gre: true) +# route plain static routes, no encapsulation. REQUIRES Crusoe port +# security to be DISABLED on this gateway's vNIC (or allowed- +# address-pairs covering vpn_remote_subnet). Without that, +# decrypted return traffic is dropped by the SDN. See the README. +# none this gateway carries no local clients (== vpn_use_gre: false) +# +# "" = gre_fou. That is the default because "route" REQUIRES Crusoe NetEng to +# disable port security on every gateway vNIC by hand, and no playbook can do +# that for you. +# +# Left as "" rather than the literal "gre_fou" so the deprecated vpn_use_gre +# spelling stays reachable: hardcoding it here would make "vpn_use_gre: false" +# silently do nothing. See "Resolve client transport" in tasks/main.yml. +vpn_client_transport: "" + +# vpn_use_gre is the deprecated alias, and it is DELIBERATELY NOT DEFAULTED. +# +# tasks/main.yml decides the transport by asking whether it was explicitly +# set. A default here makes "vpn_use_gre is defined" always true, so every +# gateway resolves to the "none" transport and carries no client traffic at +# all - the deploy reports success and nothing flows. Measured on live hosts. +# Leave it undefined. + +# GRE-over-FOU (GRE in UDP - passes firewalls that block raw GRE proto 47) +# +# MTU maths, and why 1400 rather than the 1360 this role used to ship: +# +# The gateway STRIPS the GRE/FOU wrapper before encrypting, so IPsec carries +# the client's original packet, not the wrapped one. The client's packet size +# is therefore bounded by two things: +# the XFRM interface -> vpn_xfrm_mtu (1400) +# the client->gateway hop -> uplink MTU - 36 (1500 - 36 = 1464) +# The binding constraint is 1400, so 1360 was leaving 40 bytes unused - about +# 2.9% of payload, which matters because the peer caps PACKETS per second. +# +# Lower this if the uplink MTU is below 1500, or vpn_xfrm_mtu is reduced. +# The role asserts both constraints at deploy time. +vpn_gre_mtu: 1400 +vpn_gre_key: 100 +vpn_fou_port: 9473 + +# Randomise the outer FOU source port per inner flow on the packets THIS +# gateway sends to its clients, so a client's NIC sees many RSS flows from +# one gateway instead of one. The receiver matches on destination port only. +# The client role has the matching setting for the other direction. +vpn_fou_sport_auto: true + +# For kernels whose iproute2 lacks "encap-sport auto": N FOU ports. +# Device name stays exactly "gre-vpn" while this is 1. +vpn_fou_port_count: 1 + +# How "route" transport gets decrypted return traffic past the SDN: +# port_security_disabled NetEng disabled port security / added allowed- +# address-pairs. Stateless, scales. Recommended. +# snat gateway SNATs remote->client traffic to its own +# address. No network-team dependency, but needs +# conntrack per packet and is INCOMPATIBLE with +# multi-gateway ECMP. Not for >10 Gbps. +vpn_route_return: "port_security_disabled" + +# SNAT decrypted traffic heading for the internet. Required for full-tunnel +# internet egress, but SNAT is stateful, so it CANNOT be combined with +# multi-gateway ECMP. Off by default: the shipped datapath is stateless so +# ECMP survives a failover. Turn it on only for full-tunnel internet egress +# through a SINGLE gateway. +vpn_snat_internet_egress: false + +# The multipoint GRE tunnel holds one PERMANENT neighbour entry per host in +# vpn_client_cidr. The kernel's default hard limit is 1024, which a /22 or +# larger silently exceeds. These cover up to a /19 (8190 hosts). +vpn_neigh_gc_thresh1: 8192 +vpn_neigh_gc_thresh2: 16384 +vpn_neigh_gc_thresh3: 32768 + +# Policy routing +vpn_rt_table: 100 # xfrm -> client overlay +vpn_mark_table: 200 # marked client traffic -> xfrm +vpn_fwmark: "0x64" + +# --------------------------------------------------------------------------- +# BGP (FRR) - MANDATORY for multi-gateway ECMP against Azure +# --------------------------------------------------------------------------- +# Azure requires BGP whenever several on-premises devices advertise the same +# prefixes to one VPN gateway. Static routing cannot express it. +vpn_bgp_enabled: false +vpn_bgp_local_asn: 65010 # unique per gateway VM +vpn_bgp_peer_asn: 65515 # Azure's default +vpn_bgp_advertise: [] # Crusoe prefixes - IDENTICAL on every gateway +vpn_bgp_accept: [] # prefixes to accept from the peer +vpn_bgp_peer_addrs: [] # peer BGP address per tunnel, in tunnel order +vpn_bgp_maximum_paths: 8 +vpn_bgp_keepalive: 10 +vpn_bgp_holdtime: 30 +vpn_bgp_ebgp_multihop: 8 + +# With BGP the prefixes are dynamic, so let routing decide. +vpn_traffic_selectors_any: "{{ vpn_bgp_enabled }}" + +# --------------------------------------------------------------------------- +# Performance - see README.md and AZURE-10G-GUIDE.md +# --------------------------------------------------------------------------- +# Master switch for host tuning (offloads, queues, RPS, IRQ, governor). +vpn_perf_tuning: true + +# ECMP hashing. 0 (kernel default) hashes src/dst IP only, so every flow +# between the same host pair takes ONE nexthop and the other tunnels idle. +# The single most missable line in this whole solution. +vpn_multipath_hash_policy: 1 + +# Loose reverse-path filtering - with ECMP the reverse path will not always +# match the forward nexthop. +vpn_rp_filter: 2 + +# Clamp forwarded TCP SYNs. ON by default: the one intentional behaviour +# change, because without it TCP relies on PMTUD across a path that may drop +# ICMP. +# +# "" = DERIVE from the transport actually in use, which is the only safe +# default. iptables TCPMSS --set-mss sets the MSS to exactly that value - it +# will RAISE an MSS as happily as lower it. Setting a value larger than the +# path can carry produces a silent blackhole: ICMP still works, TCP collapses +# to a few Mbit/s with massive retransmits. Measured on real hardware: +# 3.36 Mbps instead of 9 Gbps. +# +# Derived values (see vpn_mss_effective in tasks/main.yml): +# gre_fou vpn_gre_mtu - 40 (1400 - 40 = 1360) +# route/none vpn_xfrm_mtu - 40 (1400 - 40 = 1360) +# +# Both come to 1360 because vpn_gre_mtu is 1400: the gateway strips the GRE +# wrapper before encrypting, so the tunnel MTU, not the GRE MTU, is the +# binding constraint. Azure's documented "MSS 1350" is therefore safe on both +# transports - but derive it anyway, because a lower vpn_gre_mtu (a sub-1500 +# uplink) moves the limit and a pinned 1350 would then be a blackhole. +vpn_mss_clamp: true +vpn_mss_clamp_value: "" # "" = derive (recommended). A number pins it. + +# Nothing in this design needs conntrack. NOTRACK removes its per-packet cost +# AND enforces the statelessness that multi-gateway ECMP depends on. +vpn_disable_conntrack: true +# The ESTABLISHED,RELATED FORWARD rule. Must be false when conntrack is +# disabled, because a NOTRACK'd packet can never match a ctstate rule. +vpn_stateful_forward_rule: false + +# GRO MUST BE OFF on any host that RECEIVES GRE-over-FOU. Generic Receive +# Offload coalesces the inbound FOU/UDP packets on the physical NIC; the +# oversized result cannot be re-encapsulated on the way out and is dropped. +# UDP is unaffected, so the tunnel looks perfectly healthy while TCP collapses. +# +# Measured on 8 vCPU VMs across two Crusoe regions, client to client: +# ens3 GRO on -> 3.65 Mbps +# ens3 GRO off -> 4590 Mbps (~1250x) +# +# This is correctness, not tuning, so it is applied by vpn-network.sh whenever +# the transport is gre_fou - not gated behind vpn_perf_tuning. The route +# transport has no FOU and keeps GRO on. +vpn_fou_disable_gro: true + +# NIC offloads and queues. GRO is deliberately absent here; see above. +vpn_nic_offloads: "gso on tso on" +vpn_rx_queues: "" # "" = one combined queue per vCPU, capped at NIC max +vpn_adaptive_rx: true + +# Receive Packet Steering. Re-queues packets to other CPUs early in +# netif_receive_skb, before the XFRM input path. +# +# IMPORTANT, and measured: RPS hashes PER FLOW, so it spreads MANY flows over +# more CPUs than the NIC has queues. It does NOT split a single flow. One +# IPsec tunnel is one outer flow, so its ESP decrypt stays pinned to one core +# no matter what RPS is set to - observed live at 94% on one core of a 16 vCPU +# gateway with RPS enabled. The only fix for that is MORE TUNNELS. +# +# Useful on a gateway that receives many client FOU flows; useless for the +# single ESP flow arriving from the peer. +vpn_enable_rps: true + +# --------------------------------------------------------------------------- +# Receive Flow Steering - SEPARATE from RPS above, and the most valuable +# per-tunnel knob measured on Crusoe mlx5 hardware. +# --------------------------------------------------------------------------- +# RFS (rps_flow_cnt) steers a packet to the CPU where the consuming socket +# last ran. For ONE IPsec tunnel that is actively harmful: the SA's packets +# are handed between CPUs, reach xfrm_input out of order, and the ESP +# anti-replay window rejects the stragglers. The tunnel stays up, ping stays +# clean, and TCP quietly retransmits instead. +# +# Measured: one tunnel, 16 flows, 54 ms RTT, kernel 6.8, identical WAN path. +# rps_flow_cnt 4096 -> 4.67 Gbps 18,624 retransmits 46,697 replay drops +# rps_flow_cnt 0 -> 4.86 Gbps 106 retransmits 3,141 replay drops +# +4.2% throughput, 176x fewer retransmits +# +# CAVEAT, measured: that comparison is at 4.7 Gbps on kernel 6.8. On 6.11 at +# 5.7 Gbps the anti-replay rejections come back to ~1M per 20 s with RFS +# already off, because at that rate the reordering is coming from the WAN +# path, not from RFS - the no-tunnel control on the same path retransmits +# 2.57M at 9 Gbps. So RFS-off removes the reordering THIS HOST causes; it +# cannot fix what the network does. +# +# There is still no case in this design where RFS helps, so this defaults to +# true and is applied whenever vpn_perf_tuning is on. +vpn_disable_rfs: true + +# NIC interrupt coalescing. Fixed coalescing batches more packets per NAPI +# poll, which helps a single-queue tunnel a little. Measured +2.6% at +# rx-usecs 64. rx-usecs 128 measured -2.0%, so do not push it further. +# 0 = leave the driver's adaptive-rx alone. That is the safe default. +vpn_rx_usecs: 0 + +# GRO batch size. Measured +1.1%, which is inside this rig's noise band, +# so it is off by default. 0 = leave the kernel default of 8. +vpn_gro_normal_batch: 0 + +# NIC ring sizes: DELIBERATELY NOT RAISED. +# The mlx5 VF permits rx/tx 8192 and the obvious move is to take it. Measured +# on the same rig, rx/tx 8192 gave 4.24 Gbps against a 4.67 Gbps baseline - a +# 9.1% REGRESSION. A deeper ring holds more packets in flight, which widens +# reordering and feeds the anti-replay problem above. +# 0 = leave the driver default. A number sets both rx and tx. +vpn_nic_rings: 0 + +# NOT SET BY THIS ROLE, and measured so you do not have to: +# the mlx5 private flags rx_cqe_moder and rx_cqe_compress are widely +# recommended for throughput. On this VF they measured -0.2%. Nothing. + +# --------------------------------------------------------------------------- +# Kernel AES-GCM implementation check +# --------------------------------------------------------------------------- +# The kernel binds the highest-priority driver registered for +# rfc4106(gcm(aes)). Kernels before 6.11 ship only the 2010-era AES-NI +# implementation. 6.11 and later add a VAES/AVX-512 one that is far cheaper +# per byte on AMD Zen 4/5 and recent Intel. +# +# Measured: one tunnel, same two VMs, same WAN path (9.01 Gbps raw control). +# 6.8.0-78 rfc4106-gcm-aesni 4.67 Gbps mean decrypt core 83% +# 6.11.0-29 rfc4106-gcm-vaes-avx10_512 5.75 Gbps mean decrypt core 62% +# +# +23% per tunnel, and crypto fell from 22.6% of receive cycles to under 3%. +# +# The kernel play (tasks/kernel.yml) installs and, where safe, boots the right +# kernel when vpn_kernel_upgrade is true. This check is the report that +# remains when that is off, or could not act (no package, live gateway). +vpn_check_crypto_accel: true + +# Fail the play, rather than warn, when the kernel has no VAES AES-GCM driver. +# Left false so an existing deployment is not blocked by it. +vpn_require_crypto_accel: false + +# --------------------------------------------------------------------------- +# Preflight (tasks/preflight.yml, the FIRST play in site.yml) +# --------------------------------------------------------------------------- +# Nothing is configured until every check passes on every host. Replaces the +# old scripts/preflight.sh, which could be skipped and only knew about two +# hand-typed SSH targets. +# +# Checks: the five required values are filled in; the OS is apt-based; the +# apt mirrors answer; xfrm_interface/fou/ip_gre/esp4 exist; the uplink MTU is +# at least 1500; every client VM falls inside some gateway's vpn_client_cidr; +# and UDP 500 + 4500 pass in BOTH directions between gateways. +vpn_preflight: true + +# The bidirectional UDP probe. Only runs when the peer is a host in THIS +# inventory - a managed cloud gateway has nothing to run a listener on - and +# is skipped when a tunnel is already established, because an installed SA is +# stronger evidence than any probe. +vpn_preflight_udp_test: true +vpn_preflight_listen_secs: 25 + +# --------------------------------------------------------------------------- +# Automatic kernel upgrade (tasks/kernel.yml, second play in site.yml) +# --------------------------------------------------------------------------- +# ON by default, because the whole point of this role's defaults is that a +# customer runs one command and gets a correctly performing gateway. +# +# It is conservative about when it acts. It does nothing at all unless ALL of: +# * vpn_kernel_upgrade is true +# * the CPU actually advertises the "vaes" flag +# * the running kernel is NOT already bound to a *vaes* AES-GCM driver +# * the kernel package is available from the host's mirrors +# +# On a FIRST deploy that means once, before any VPN config exists, so the +# reboot interrupts nothing. Two further guards make it safe to leave on: +# * a host that already carries a VPN (swanctl config or established SA) +# gets the kernel INSTALLED but is NOT rebooted unless +# vpn_kernel_reboot_live is true - the note it prints tells you how +# * a host already running the newest installed kernel of the requested +# series is never rebooted again, so a CPU whose 6.11 still binds AES-NI +# (vaes without AVX512BW/VL) warns once instead of looping +vpn_kernel_upgrade: true + +# 6.11 is the first release with VAES/AVX-512 AES-GCM and is what was measured +# (+23% per tunnel). 6.14 also carries it. +# +# DO NOT set this to 7.0. Measured on this hardware, 7.0 has the same fast +# crypto and was still 2.4x SLOWER end to end (2.16 vs 5.75 Gbps): a perf +# profile showed the cycles moving into fib_rules_lookup, nft_meta_store_ifname, +# xfrm_sk_policy_lookup, pskb_expand_head and the srso_safe_ret speculation +# mitigation, which 6.11 reports as "Not affected" on the same CPU. +vpn_kernel_package: "linux-image-generic-6.11" + +# Reboot to activate it. With this false the kernel is installed and made +# bootable, and the host keeps running the old one until you reboot yourself. +vpn_kernel_reboot: true +vpn_kernel_reboot_timeout: 900 + +# Allow the reboot on a host that ALREADY carries a VPN. Off by default because +# it drops every flow through that gateway for the duration. Turn it on for a +# planned window, one gateway at a time, so the others keep carrying traffic. +vpn_kernel_reboot_live: false + +vpn_netdev_budget: 600 +vpn_netdev_budget_usecs: 8000 +vpn_irq_affinity: true # pin NIC IRQs + xps_cpus, stop irqbalance moving them +vpn_cpu_governor: "performance" # "" to leave the host alone + +vpn_rmem_max: 268435456 +vpn_wmem_max: 268435456 -# Set to true + vpn_client_cidr for Crusoe (GRE-over-FOU mode) -vpn_use_gre: false -# vpn_client_cidr: "" +# Assertions, not tuning. Turn silent misconfiguration into a loud failure. +vpn_assert_aes_ni: true # fail if the CPU has no AES-NI +vpn_assert_gcm_plugin: true # warn if strongSwan cannot do GCM +vpn_assert_negotiated_esp: true # warn if the live SAs are not using GCM diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/handlers/main.yml b/strongswan-ipsec/ansible/roles/vpn_gateway/handlers/main.yml index 00ef5b4..32103c5 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/handlers/main.yml +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/handlers/main.yml @@ -10,6 +10,12 @@ swanctl --load-all 2>/dev/null || true changed_when: false +- name: restart frr + ansible.builtin.systemd: + name: frr + state: restarted + failed_when: false + - name: reload systemd ansible.builtin.systemd: daemon_reload: true diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/kernel.yml b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/kernel.yml new file mode 100644 index 0000000..91a8cb1 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/kernel.yml @@ -0,0 +1,394 @@ +--- +# ============================================================================= +# Ensure the gateway runs a kernel with VAES/AVX-512 AES-GCM +# +# The kernel binds the highest-priority driver registered for +# rfc4106(gcm(aes)). Kernels before 6.11 ship only the 2010-era AES-NI +# implementation. 6.11 added a VAES/AVX-512 one. +# +# Measured, one tunnel, same two VMs, same WAN path (9.01 Gbps raw control +# either side of the change): +# +# 6.8.0-78 rfc4106-gcm-aesni 4.67 Gbps mean decrypt core 83% +# 6.11.0-29 rfc4106-gcm-vaes-avx10_512 5.75 Gbps mean decrypt core 62% +# +# +23% per tunnel, and crypto falls from 22.6% of receive cycles to under 3%. +# +# WHEN IT ACTS, precisely - because a reboot is the most disruptive thing a +# playbook can do to a gateway: +# +# * only when vpn_kernel_upgrade is true AND the CPU advertises "vaes" +# * only when the bound AES-GCM driver is not already a *vaes* one +# * NEVER when the host is already running the newest installed kernel of +# the requested series. Rebooting again cannot change which driver binds, +# so a CPU whose 6.11 still picks AES-NI (vaes without AVX512BW/VL) gets a +# warning once instead of a reboot on every run. +# * it INSTALLS on any host, but REBOOTS only a host with no VPN on it yet. +# If a swanctl config or an established SA exists, the reboot is deferred +# and announced - unless vpn_kernel_reboot_live is true. A first deploy +# therefore reboots before any VPN config exists and interrupts nothing; +# an upgrade of a live gateway installs the kernel and waits for you. +# ============================================================================= + +- name: Probe kernel, crypto driver and VPN state + ansible.builtin.shell: + cmd: | + modprobe aesni_intel 2>/dev/null || true + # /proc/crypto also lists internal instances ("__rfc4106(...)") and + # wrappers ("seqiv(rfc4106(...))"). Anchoring the name selects the entry + # consumers actually bind. sed BRE treats "(" literally, so no escaping. + drv=$(sed -n "/^name .*: rfc4106(gcm(aes))$/,+1p" /proc/crypto \ + | sed -n "s/^driver *: *//p" | head -1) + vaes=no; grep -qw vaes /proc/cpuinfo && vaes=yes + # Newest INSTALLED kernel of the requested series (the "6.11" in + # linux-image-generic-6.11). Without a series in the package name, the + # newest kernel of any series. + series=$(printf '%s' "{{ vpn_kernel_package }}" | grep -oE '[0-9]+\.[0-9]+' | head -1) + if [ -n "$series" ]; then + target=$(ls -1 /boot/vmlinuz-"$series".* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) + else + target=$(ls -1 /boot/vmlinuz-* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) + fi + # Is there a VPN on this host already? Either signal means a reboot + # would drop live traffic. + sas=0 + if command -v swanctl >/dev/null 2>&1; then + sas=$(swanctl --list-sas 2>/dev/null | grep -c INSTALLED || true) + fi + conf=no; [ -f /etc/swanctl/conf.d/site-vpn.conf ] && conf=yes + echo "${drv:-unknown}|${vaes}|$(uname -r)|${target:-none}|${sas:-0}|${conf}" + register: vpn_kern_probe + changed_when: false + check_mode: false + tags: [kernel, setup] + +- name: Record what the probe found + ansible.builtin.set_fact: + vpn_gcm_driver: "{{ (vpn_kern_probe.stdout | trim).split('|')[0] }}" + vpn_cpu_has_vaes: "{{ (vpn_kern_probe.stdout | trim).split('|')[1] == 'yes' }}" + vpn_running_kernel: "{{ (vpn_kern_probe.stdout | trim).split('|')[2] }}" + vpn_kern_installed_target: "{{ (vpn_kern_probe.stdout | trim).split('|')[3] }}" + vpn_kern_live_sas: "{{ (vpn_kern_probe.stdout | trim).split('|')[4] | int }}" + vpn_kern_conf_present: "{{ (vpn_kern_probe.stdout | trim).split('|')[5] == 'yes' }}" + tags: [kernel, setup] + +- name: Decide what to do + ansible.builtin.set_fact: + # Already on the newest installed kernel of the requested series: another + # reboot cannot change the driver, so never loop. + vpn_kernel_already_on_target: >- + {{ vpn_kern_installed_target != 'none' + and vpn_kern_installed_target == vpn_running_kernel }} + # A VPN exists here. Installing is harmless; rebooting is not. + vpn_kernel_live: >- + {{ (vpn_kern_live_sas | int) > 0 or (vpn_kern_conf_present | bool) }} + tags: [kernel, setup] + +- name: Decide whether a kernel change is needed + ansible.builtin.set_fact: + vpn_kernel_change_needed: >- + {{ (vpn_kernel_upgrade | bool) + and (vpn_cpu_has_vaes | bool) + and ('vaes' not in vpn_gcm_driver) + and not (vpn_kernel_already_on_target | bool) }} + tags: [kernel, setup] + +- name: Report the decision + ansible.builtin.debug: + msg: >- + kernel={{ vpn_running_kernel }} driver={{ vpn_gcm_driver }} + cpu_vaes={{ vpn_cpu_has_vaes }} live_vpn={{ vpn_kernel_live }} + {%- if 'vaes' in vpn_gcm_driver %} + -> already on the VAES fast path, nothing to do. + {%- elif not (vpn_cpu_has_vaes | bool) %} + -> this CPU has no VAES, so AES-NI is already the best available. + {%- elif not (vpn_kernel_upgrade | bool) %} + -> vpn_kernel_upgrade is false; staying on the slow crypto path. + {%- elif vpn_kernel_already_on_target | bool %} + -> WARNING: already running {{ vpn_kern_installed_target }}, the newest + installed kernel of the requested series, and it still binds + {{ vpn_gcm_driver }}. This CPU's VAES cannot be used by this kernel + (it needs AVX512BW/VL too). Not rebooting again; roughly 23% per + tunnel is left on the table. + {%- elif vpn_kernel_live | bool and not (vpn_kernel_reboot_live | bool) %} + -> installing {{ vpn_kernel_package }}; NOT rebooting, because this host + already carries a VPN ({{ vpn_kern_live_sas }} SA(s) established). See the + note at the end of this play. + {%- else %} + -> installing {{ vpn_kernel_package }} and rebooting (worth ~+23%/tunnel). + {%- endif %} + tags: [kernel, setup] + +# --------------------------------------------------------------------------- +# Install +# --------------------------------------------------------------------------- + +- name: Refresh the package index + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + when: vpn_kernel_change_needed | bool + tags: [kernel, setup] + +- name: Check the kernel package is available + ansible.builtin.shell: + cmd: | + apt-cache policy {{ vpn_kernel_package }} 2>/dev/null \ + | sed -n 's/^ *Candidate: *//p' | head -1 + register: vpn_kern_candidate + changed_when: false + when: vpn_kernel_change_needed | bool + tags: [kernel, setup] + +# A missing package is a "carry on slower", not a "stop the deployment". +- name: Skip the upgrade when the package is unavailable + ansible.builtin.set_fact: + vpn_kernel_change_needed: false + when: + - vpn_kernel_change_needed | bool + - (vpn_kern_candidate.stdout | default('') | trim) in ['', '(none)'] + tags: [kernel, setup] + +- name: Warn that the kernel package is unavailable + ansible.builtin.debug: + msg: >- + {{ vpn_kernel_package }} is not available on this host, so the gateway + stays on {{ vpn_running_kernel }} with the slower AES-NI crypto. This + costs roughly 23% per tunnel but breaks nothing. Set + vpn_kernel_package to a kernel your mirrors carry (6.11 or newer), or + vpn_kernel_upgrade=false to silence this. + when: + - vpn_kernel_upgrade | bool + - vpn_cpu_has_vaes | bool + - "'vaes' not in vpn_gcm_driver" + - not (vpn_kernel_already_on_target | bool) + - not (vpn_kernel_change_needed | bool) + - (vpn_kern_candidate.stdout | default('') | trim) in ['', '(none)'] + tags: [kernel, setup] + +- name: Install the VAES-capable kernel + ansible.builtin.apt: + name: "{{ vpn_kernel_package }}" + state: present + when: vpn_kernel_change_needed | bool + tags: [kernel, setup] + +# --------------------------------------------------------------------------- +# Make it bootable +# +# MEASURED FAILURE: on both test VMs the package install left +# /boot/initrd.img pointing at an initramfs that was never generated. GRUB +# skips a kernel with no initrd, so the node rebooted straight back onto the +# old kernel and looked like the upgrade had simply not worked. Everything +# below exists to catch that before spending a reboot on it. +# --------------------------------------------------------------------------- + +- name: Prepare the new kernel for boot + ansible.builtin.shell: + cmd: | + set -u + # Same series filter as the probe: newest installed kernel of the + # requested series, so a newer kernel of another series that happens to + # be present is never picked by accident. + series=$(printf '%s' "{{ vpn_kernel_package }}" | grep -oE '[0-9]+\.[0-9]+' | head -1) + if [ -n "$series" ]; then + ver=$(ls -1 /boot/vmlinuz-"$series".* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) + else + ver=$(ls -1 /boot/vmlinuz-* 2>/dev/null | sed 's|.*/vmlinuz-||' | sort -V | tail -1) + fi + if [ -z "$ver" ]; then echo "NOVER|absent|no|unknown"; exit 0; fi + + # The bug: kernel present, initramfs absent. + if [ ! -f "/boot/initrd.img-$ver" ]; then + update-initramfs -c -k "$ver" >/dev/null 2>&1 || true + fi + built=absent; [ -f "/boot/initrd.img-$ver" ] && built=present + + update-grub >/dev/null 2>&1 || true + + # Is the new kernel actually in the menu at all? + inmenu=no + grep -q "vmlinuz-$ver" /boot/grub/grub.cfg 2>/dev/null && inmenu=yes + + # Which kernel will actually boot? That depends on GRUB_DEFAULT, not just + # on menu order: + # unset / 0 / "Ubuntu" the first menuentry, which Ubuntu generates for + # the newest kernel -> first vmlinuz in grub.cfg + # saved whatever grubenv remembers + # "...Linux " an explicitly pinned kernel + # A pinned or saved default pointing at the OLD kernel is exactly how a + # reboot lands back where it started, so read it rather than assume. + first=$(grep -m1 -oE "/boot/vmlinuz-[^ ]+" /boot/grub/grub.cfg 2>/dev/null \ + | sed "s|.*/vmlinuz-||") + gd=$(sed -n 's/^GRUB_DEFAULT=//p' /etc/default/grub 2>/dev/null | tr -d '"' | tail -1) + case "${gd:-}" in + ""|0|Ubuntu) + defk="$first" ;; + saved) + se=$(grub-editenv list 2>/dev/null | sed -n 's/^saved_entry=//p') + defk=$(printf '%s' "$se" | grep -oE 'Linux [^ ]+' | awk '{print $2}') + [ -z "$defk" ] && defk="$first" ;; + *) + defk=$(printf '%s' "$gd" | grep -oE 'Linux [^ ]+' | awk '{print $2}') + [ -z "$defk" ] && defk="pinned:$gd" ;; + esac + + echo "$ver|$built|$inmenu|${defk:-unknown}" + register: vpn_kern_prep + changed_when: true + when: vpn_kernel_change_needed | bool + tags: [kernel, setup] + +- name: Parse the boot preparation + ansible.builtin.set_fact: + vpn_kern_target: "{{ (vpn_kern_prep.stdout | trim).split('|')[0] }}" + vpn_kern_initrd: "{{ (vpn_kern_prep.stdout | trim).split('|')[1] }}" + vpn_kern_inmenu: "{{ (vpn_kern_prep.stdout | trim).split('|')[2] }}" + vpn_kern_default: "{{ (vpn_kern_prep.stdout | trim).split('|')[3] }}" + when: + - vpn_kernel_change_needed | bool + - "'|' in (vpn_kern_prep.stdout | default(''))" + tags: [kernel, setup] + +- name: Refuse to reboot into a kernel GRUB cannot boot + ansible.builtin.assert: + that: + - vpn_kern_target != 'NOVER' + - vpn_kern_initrd == 'present' + - vpn_kern_inmenu == 'yes' + fail_msg: >- + Kernel {{ vpn_kern_target }} is installed but not bootable + (initramfs={{ vpn_kern_initrd }}, in GRUB menu={{ vpn_kern_inmenu }}). + Rebooting now would land back on {{ vpn_running_kernel }}. Fix with: + sudo update-initramfs -c -k {{ vpn_kern_target }} && sudo update-grub + success_msg: >- + {{ vpn_kern_target }} is bootable (initramfs present, in GRUB menu, + default entry boots {{ vpn_kern_default }}). + when: vpn_kernel_change_needed | bool + tags: [kernel, setup] + +# Only touch GRUB_DEFAULT if the default entry would boot the wrong kernel. +# Most hosts need nothing here, and a customer's GRUB config is not ours to +# rewrite speculatively. +- name: Back up /etc/default/grub before changing the default entry + ansible.builtin.copy: + src: /etc/default/grub + dest: /etc/default/grub.vpn-backup + remote_src: true + force: false + mode: "0644" + when: + - vpn_kernel_change_needed | bool + - vpn_kern_default != vpn_kern_target + tags: [kernel, setup] + +- name: Point GRUB at the new kernel + ansible.builtin.shell: + cmd: | + if grep -q '^GRUB_DEFAULT=' /etc/default/grub; then + sed -i 's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux {{ vpn_kern_target }}"|' \ + /etc/default/grub + else + printf '%s\n' 'GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux {{ vpn_kern_target }}"' \ + >> /etc/default/grub + fi + update-grub >/dev/null 2>&1 + grep '^GRUB_DEFAULT' /etc/default/grub + register: vpn_kern_grubset + when: + - vpn_kernel_change_needed | bool + - vpn_kern_default != vpn_kern_target + tags: [kernel, setup] + +# --------------------------------------------------------------------------- +# Reboot - only a host that is not yet carrying a VPN - and prove it worked +# --------------------------------------------------------------------------- + +- name: Decide whether this host may be rebooted now + ansible.builtin.set_fact: + vpn_kernel_do_reboot: >- + {{ (vpn_kernel_change_needed | bool) and (vpn_kernel_reboot | bool) + and ((not (vpn_kernel_live | bool)) or (vpn_kernel_reboot_live | bool)) }} + tags: [kernel, setup] + +- name: Explain that a reboot is about to happen + ansible.builtin.debug: + msg: >- + Rebooting into {{ vpn_kern_target }}. + {%- if vpn_kernel_live | bool %} + This host HAS a live VPN and vpn_kernel_reboot_live is true, so traffic + through it will drop for the duration of the reboot. + {%- else %} + No VPN config exists on this host yet, so nothing is interrupted. + {%- endif %} + when: vpn_kernel_do_reboot | bool + tags: [kernel, setup] + +- name: Reboot into the new kernel + ansible.builtin.reboot: + reboot_timeout: "{{ vpn_kernel_reboot_timeout }}" + msg: "Rebooting for a VAES-capable kernel (strongswan-ipsec)" + when: vpn_kernel_do_reboot | bool + tags: [kernel, setup] + +- name: Re-probe the AES-GCM driver after reboot + ansible.builtin.shell: + cmd: | + modprobe aesni_intel 2>/dev/null || true + drv=$(sed -n "/^name .*: rfc4106(gcm(aes))$/,+1p" /proc/crypto \ + | sed -n "s/^driver *: *//p" | head -1) + echo "${drv:-unknown}|$(uname -r)" + register: vpn_kern_after + changed_when: false + when: vpn_kernel_do_reboot | bool + tags: [kernel, setup] + +- name: Confirm the fast crypto path is live + ansible.builtin.assert: + that: "'vaes' in (vpn_kern_after.stdout | default(''))" + fail_msg: >- + Rebooted but still on {{ (vpn_kern_after.stdout | default('?')).split('|')[0] }} + running {{ (vpn_kern_after.stdout | default('?|?')).split('|')[1] }}. + Either GRUB booted the wrong kernel (check "grep menuentry + /boot/grub/grub.cfg" and /etc/default/grub - the original is at + /etc/default/grub.vpn-backup) or this CPU's VAES is unusable by this + kernel. The next run will NOT reboot again: it detects that the + requested kernel is already running. Deployment continues on the slower + path, roughly 23% per tunnel down. + success_msg: >- + Now running {{ (vpn_kern_after.stdout | default('?|?')).split('|')[1] }} + on {{ (vpn_kern_after.stdout | default('?')).split('|')[0] }} - the VAES + fast path. + when: vpn_kernel_do_reboot | bool + # Worth shouting about, but must not block a deployment that would + # otherwise work. The already-on-target guard above stops this recurring. + ignore_errors: true + tags: [kernel, setup] + +- name: Note that the reboot was deferred because this gateway is live + ansible.builtin.debug: + msg: >- + {{ vpn_kern_target }} is installed and bootable, but this host already + carries a VPN ({{ vpn_kern_live_sas }} SA(s) established, + config present={{ vpn_kern_conf_present }}), so it was NOT rebooted. + It keeps running {{ vpn_running_kernel }} with the slower crypto until + you reboot it - in a maintenance window, one gateway at a time so the + others keep carrying traffic - or re-run with + "-e vpn_kernel_reboot_live=true" to let this play do it. + when: + - vpn_kernel_change_needed | bool + - vpn_kernel_reboot | bool + - vpn_kernel_live | bool + - not (vpn_kernel_reboot_live | bool) + tags: [kernel, setup] + +- name: Note that a reboot is still pending + ansible.builtin.debug: + msg: >- + {{ vpn_kern_target }} is installed and bootable but vpn_kernel_reboot is + false, so this host is still running {{ vpn_running_kernel }} with the + slower crypto. Reboot when convenient to collect the ~23%. + when: + - vpn_kernel_change_needed | bool + - not (vpn_kernel_reboot | bool) + tags: [kernel, setup] diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/main.yml b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/main.yml index 14c52b0..e6fcf11 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/main.yml +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/main.yml @@ -2,20 +2,281 @@ # ============================================================================= # vpn_gateway role # -# Two modes controlled by vpn_use_gre: -# true (Crusoe): Multipoint GRE tunnel + CIDR-based neighbor table -# false (Azure/GCP/AWS): Standard IP forwarding via cloud NIC-level flag +# Terminates N IPsec tunnels to a remote site and delivers traffic to local +# Crusoe clients. Three client transports, selected by vpn_client_transport: +# +# gre_fou GRE-over-FOU overlay (default; == the old vpn_use_gre: true) +# route plain static routes - needs Crusoe port security DISABLED +# none no local clients; forward this gateway's own LAN subnet +# +# With every new variable at its default this behaves exactly as the +# single-tunnel version always did. See README.md, and AZURE-10G-GUIDE.md for +# the recipe that reaches 10 Gbps with the fewest VMs. # ============================================================================= -# ---------- Detect environment ---------- +# --------------------------------------------------------------------------- +# Detect environment and derive the shape of this deployment +# --------------------------------------------------------------------------- - name: Detect primary network interface and local IP ansible.builtin.set_fact: vpn_primary_iface: "{{ ansible_default_ipv4.interface }}" vpn_local_gw_ip: "{{ ansible_default_ipv4.address }}" - tags: [setup, teardown] + tags: [setup, teardown, verify] + +# Precedence: vpn_client_transport, then the deprecated vpn_use_gre, then the +# gre_fou default. vpn_use_gre is only consulted when it was EXPLICITLY set - +# so "vpn_use_gre: false" still selects the "none" transport instead of being +# silently overridden by the default. +- name: Resolve client transport + ansible.builtin.set_fact: + vpn_transport: >- + {{ vpn_client_transport if vpn_client_transport + else ('none' if (vpn_use_gre is defined and not (vpn_use_gre | bool)) + else 'gre_fou') }} + tags: [setup, teardown, verify] + +- name: Derive transport booleans and traffic selectors + ansible.builtin.set_fact: + vpn_serves_clients: "{{ vpn_transport in ['gre_fou', 'route'] }}" + vpn_child_local_ts: >- + {{ '0.0.0.0/0' if (vpn_traffic_selectors_any | bool) else vpn_local_subnet }} + vpn_child_remote_ts: >- + {{ '0.0.0.0/0' if (vpn_traffic_selectors_any | bool) else vpn_remote_subnet }} + tags: [setup, teardown, verify] + +# vpn_tunnel_count 0 means "one tunnel per remote outer address". An Azure +# active-active gateway publishes two public IPs, so listing both in +# vpn_remote_addrs yields two tunnels with nothing else to set. Two tunnels to +# the SAME address share one outer 4-tuple, one RX queue and one CPU, so they +# would buy nothing - hence counting addresses rather than hardcoding 2. +- name: Derive the effective tunnel count + ansible.builtin.set_fact: + vpn_tunnel_count_effective: >- + {{ (vpn_tunnel_count | int) if (vpn_tunnel_count | int) > 0 + else ([(vpn_remote_addrs | length), 1] | max) }} + tags: [setup, teardown, verify] + +- name: Build the tunnel list + ansible.builtin.set_fact: + vpn_tunnels: "{{ lookup('template', 'tunnels.json.j2') | from_json }}" + tags: [setup, teardown, verify] + +# TCPMSS --set-mss RAISES an MSS as readily as it lowers one, so a value +# larger than the client path can carry is a silent TCP blackhole. +- name: Validate the GRE overlay MTU against both constraints + ansible.builtin.assert: + that: + - (vpn_gre_mtu | int) <= (vpn_xfrm_mtu | int) + - (vpn_gre_mtu | int) + 36 <= (ansible_facts[vpn_primary_iface].mtu | int) + fail_msg: >- + vpn_gre_mtu={{ vpn_gre_mtu }} does not fit. It must be <= + vpn_xfrm_mtu ({{ vpn_xfrm_mtu }}), because the gateway strips the GRE + wrapper before encrypting and the original packet then has to pass the + XFRM interface; and <= uplink MTU + ({{ ansible_facts[vpn_primary_iface].mtu }}) minus 36 bytes of + IP+UDP+GRE, because the wrapped packet has to cross the local fabric. + when: vpn_transport == 'gre_fou' + tags: [setup, verify] + +- name: Derive the safe MSS for this transport + ansible.builtin.set_fact: + vpn_mss_max: >- + {{ ((vpn_gre_mtu | int) if vpn_transport == 'gre_fou' + else (vpn_xfrm_mtu | int)) - 40 }} + tags: [setup, verify] + +- name: Set the effective MSS + ansible.builtin.set_fact: + vpn_mss_effective: >- + {{ vpn_mss_clamp_value if vpn_mss_clamp_value else vpn_mss_max }} + tags: [setup, verify] + +- name: Refuse an MSS larger than the path can carry + ansible.builtin.assert: + that: (vpn_mss_effective | int) <= (vpn_mss_max | int) + fail_msg: >- + vpn_mss_clamp_value={{ vpn_mss_clamp_value }} exceeds the largest MSS + this path can carry ({{ vpn_mss_max }} for transport {{ vpn_transport }}). + iptables TCPMSS --set-mss would RAISE the MSS to that value and every + full-size segment would be dropped: ICMP keeps working, TCP collapses. + Leave vpn_mss_clamp_value empty to derive it. + when: vpn_mss_clamp | bool + tags: [setup, verify] + +- name: Decide whether the ECMP health timer runs + ansible.builtin.set_fact: + vpn_ecmp_healthcheck_effective: >- + {{ vpn_ecmp_healthcheck if (vpn_ecmp_healthcheck | string) != '' + else ((vpn_tunnels | length) > 1 or (vpn_serves_clients | bool)) }} + tags: [setup, verify] -# ---------- Install ---------- +- name: Derive per-deployment strongSwan behaviour + ansible.builtin.set_fact: + # "trap" defers the SA until the first matching packet, so tunnels come + # up one at a time as traffic hashes to them - and the triggering flows + # are dropped meanwhile. Anything serving clients, or with more than one + # tunnel, wants them up front. + vpn_start_action_effective: >- + {{ vpn_start_action if vpn_start_action + else ('start' if ((vpn_tunnels | length) > 1 or (vpn_serves_clients | bool)) + else 'trap') }} + tags: [setup, verify] + +# --------------------------------------------------------------------------- +# Validate - turn silent misconfiguration into a loud failure +# --------------------------------------------------------------------------- + +- name: Validate configuration + ansible.builtin.assert: + that: + - (vpn_tunnel_count_effective | int) >= 1 + - (vpn_tunnel_count_effective | int) <= 32 + - vpn_transport in ['gre_fou', 'route', 'none'] + - not (vpn_serves_clients | bool) or (vpn_client_cidr is defined) + - vpn_route_return in ['port_security_disabled', 'snat'] + - vpn_crypto_profile in vpn_crypto_profiles + fail_msg: >- + Invalid vpn_gateway configuration. tunnel_count must be 1-32, + vpn_client_transport must be gre_fou/route/none, vpn_client_cidr is + required for gre_fou and route, and vpn_crypto_profile must be one of + {{ vpn_crypto_profiles.keys() | list }}. + tags: [setup, verify] + +# A replay window the kernel refuses leaves the SA uninstalled and the tunnel +# permanently down, which is much worse than a window that is merely small. +# 1024 is measured good; 32768 is measured to fail outright. +- name: Validate the ESP anti-replay window + ansible.builtin.assert: + that: + - (vpn_replay_window | int) >= 0 + - (vpn_replay_window | int) <= 4096 + fail_msg: >- + vpn_replay_window must be 0-4096. Measured on Crusoe: 1024 installs and + works, 32768 makes the SA fail to install so the tunnel never comes up. + 0 disables anti-replay entirely, which is a real security reduction on a + public-internet tunnel - do not use it to chase throughput. Prefer + vpn_disable_rfs=true, which removes the reordering at its source. + tags: [setup, verify] + +# These three combinations each produce a datapath that looks configured and +# silently drops traffic, so they are hard failures rather than warnings. +- name: Validate the conntrack / SNAT interlocks + ansible.builtin.assert: + that: + - not (vpn_disable_conntrack | bool) or not (vpn_stateful_forward_rule | bool) + - not (vpn_disable_conntrack | bool) or not (vpn_snat_internet_egress | bool) + - not (vpn_route_return == 'snat') or not (vpn_disable_conntrack | bool) + fail_msg: >- + Conntrack interlock violated. NOTRACK'd packets can never match a + ctstate rule and can never be SNAT'd, so vpn_disable_conntrack=true + requires vpn_stateful_forward_rule=false, vpn_snat_internet_egress=false + and vpn_route_return != snat. Note also that SNAT is stateful, so it + cannot be combined with multi-gateway ECMP at all - see the + "Known limitations" section of README.md. + tags: [setup, verify] + +- name: Validate BGP configuration + ansible.builtin.assert: + that: + - vpn_bgp_advertise | length > 0 + - vpn_bgp_accept | length > 0 + - (vpn_bgp_peer_addrs | length) >= (vpn_tunnels | length) + fail_msg: >- + vpn_bgp_enabled requires vpn_bgp_advertise (prefixes to announce - + IDENTICAL on every gateway), vpn_bgp_accept (prefixes to accept), and + one entry in vpn_bgp_peer_addrs per tunnel ({{ vpn_tunnels | length }} + needed, {{ vpn_bgp_peer_addrs | length }} given). Read Azure's peer + addresses from the gateway's bgpSettings.bgpPeeringAddresses. + when: vpn_bgp_enabled | bool + tags: [setup, verify] + +# The failure this catches looks like success: N tunnels, all established, all +# hashing to the same outer flow, so one RX queue and no more throughput. +- name: "WARNING: tunnel fan-out with no outer flow entropy" + ansible.builtin.debug: + msg: >- + WARNING: {{ vpn_tunnel_count_effective }} tunnels requested but both + vpn_local_addrs and vpn_remote_addrs resolve to a single address. With + NAT-T both UDP ports are pinned to 4500, so every tunnel will share one + outer 4-tuple, land on one RX queue, and perform exactly like a single + tunnel. For an Azure peer in active-active mode, set vpn_remote_addrs to + BOTH gateway instance public IPs. + when: + - (vpn_tunnel_count_effective | int) > 1 + - (vpn_local_addrs | length) <= 1 + - (vpn_remote_addrs | length) <= 1 + tags: [setup, verify] + +- name: "WARNING: route transport needs port security disabled" + ansible.builtin.debug: + msg: >- + vpn_client_transport=route forwards decrypted packets whose source is a + REMOTE private IP out of this gateway's vNIC. Crusoe port security drops + those. Confirm NetEng has disabled port security on this VM's vNIC (or + added allowed-address-pairs covering {{ vpn_remote_subnet }}) before + expecting return traffic to arrive. + when: vpn_transport == 'route' and vpn_route_return == 'port_security_disabled' + tags: [setup, verify] + +# Multipoint GRE has no signalling, so the gateway needs one PERMANENT +# neighbour entry per host in vpn_client_cidr. That does not scale: a /20 is +# 4094 entries (fine, ~seconds), a /16 is 65534 (exceeds the neighbour table +# and takes minutes). This is a real limit of the gre_fou transport. +- name: Size the GRE neighbour table + ansible.builtin.set_fact: + vpn_neigh_needed: >- + {{ (2 ** (32 - (vpn_client_cidr | regex_replace('^.*/', '') | int))) - 2 }} + when: vpn_transport == 'gre_fou' + tags: [setup, verify] + +- name: Refuse a client CIDR too large for the neighbour table + ansible.builtin.assert: + that: (vpn_neigh_needed | int) < (vpn_neigh_gc_thresh3 | int) + fail_msg: >- + vpn_client_cidr {{ vpn_client_cidr }} needs {{ vpn_neigh_needed }} + permanent GRE neighbour entries, which exceeds + net.ipv4.neigh.default.gc_thresh3 ({{ vpn_neigh_gc_thresh3 }}). The table + would overflow and an arbitrary subset of clients would silently never + work. Either narrow vpn_client_cidr to the range that actually holds + clients, raise vpn_neigh_gc_thresh1/2/3, or switch to + vpn_client_transport=route, which needs no neighbour table at all. + when: vpn_transport == 'gre_fou' + tags: [setup, verify] + +- name: "WARNING: large GRE neighbour table" + ansible.builtin.debug: + msg: >- + vpn_client_cidr {{ vpn_client_cidr }} means {{ vpn_neigh_needed }} + permanent neighbour entries on every gateway. That works, but it adds + deploy time and memory on each gateway. The route transport needs none + of it. + when: + - vpn_transport == 'gre_fou' + - (vpn_neigh_needed | int) > 8000 + tags: [setup, verify] + +- name: Check the CPU has AES-NI + ansible.builtin.command: grep -qw aes /proc/cpuinfo + register: vpn_aesni + changed_when: false + failed_when: false + tags: [setup, verify] + +- name: Require AES-NI + ansible.builtin.assert: + that: vpn_aesni.rc == 0 + fail_msg: >- + This CPU reports no AES-NI. AES-GCM will fall back to a generic + implementation roughly an order of magnitude slower, and the throughput + targets in AZURE-10G-GUIDE.md are unreachable. Set + vpn_assert_aes_ni=false to proceed anyway. + when: vpn_assert_aes_ni | bool + tags: [setup, verify] + +# --------------------------------------------------------------------------- +# Install +# --------------------------------------------------------------------------- - name: Install strongSwan (Debian/Ubuntu) ansible.builtin.apt: @@ -24,8 +285,12 @@ - strongswan-swanctl - charon-systemd - libcharon-extra-plugins + # GCM lives here on Debian/Ubuntu. Without it the gcmaes256 profiles + # fail with NO_PROPOSAL_CHOSEN and nothing says why. + - libstrongswan-standard-plugins - iptables - iproute2 + - python3 state: present update_cache: true cache_valid_time: 3600 @@ -34,13 +299,28 @@ - name: Install strongSwan (RHEL/Fedora) ansible.builtin.dnf: - name: [strongswan, iptables, iproute] + name: [strongswan, iptables, iproute, python3] state: present when: ansible_os_family == "RedHat" tags: [setup] -# Stop strongswan-starter immediately after install — it auto-starts -# charon on package install and conflicts with charon-systemd. +- name: Install FRR + ansible.builtin.package: + name: frr + state: present + when: vpn_bgp_enabled | bool + tags: [setup] + +- name: Install tuning and diagnostic tools + ansible.builtin.package: + name: [ethtool, sysstat, conntrack] + state: present + failed_when: false + when: vpn_perf_tuning | bool + tags: [setup] + +# Stop strongswan-starter immediately after install - it auto-starts charon on +# package install and conflicts with charon-systemd. - name: Disable strongswan-starter (conflicts with charon-systemd) ansible.builtin.systemd: name: strongswan-starter @@ -49,9 +329,11 @@ failed_when: false tags: [setup] -# ---------- Kernel parameters ---------- +# --------------------------------------------------------------------------- +# Kernel parameters +# --------------------------------------------------------------------------- -- name: Configure sysctl (ip_forward, no redirects) +- name: Configure sysctl ansible.builtin.template: src: sysctl-vpn.conf.j2 dest: /etc/sysctl.d/99-vpn-gateway.conf @@ -59,127 +341,148 @@ notify: reload sysctl tags: [setup] +# "sysctl --system" applies /etc/sysctl.conf LAST, after everything in +# /etc/sysctl.d/ - and many cloud images ship a tuned /etc/sysctl.conf. So our +# drop-in is applied and then silently overridden in the running system, while +# at boot (systemd-sysctl, lexical order) it wins. Re-applying our own file +# last makes the running state match both our intent and the post-reboot state. - name: Apply sysctl now - ansible.builtin.command: sysctl --system + ansible.builtin.shell: | + sysctl --system >/dev/null + sysctl -p /etc/sysctl.d/99-vpn-gateway.conf >/dev/null changed_when: false tags: [setup] -# ---------- XFRM interface (IPsec) ---------- - -- name: Remove stale XFRM interface - ansible.builtin.command: ip link del {{ vpn_xfrm_iface }} - changed_when: true - failed_when: false - tags: [setup] - -- name: Create XFRM interface +- name: Read back the sysctls that ECMP depends on ansible.builtin.shell: | - ip link add {{ vpn_xfrm_iface }} type xfrm dev {{ vpn_primary_iface }} if_id {{ vpn_xfrm_if_id }} - ip link set {{ vpn_xfrm_iface }} mtu {{ vpn_xfrm_mtu }} - ip link set {{ vpn_xfrm_iface }} up - changed_when: true - tags: [setup] - -- name: Route remote subnet through XFRM - ansible.builtin.command: ip route replace {{ vpn_remote_subnet }} dev {{ vpn_xfrm_iface }} - changed_when: true - when: vpn_remote_subnet != "0.0.0.0/0" - tags: [setup] + set -e + echo "forward=$(sysctl -n net.ipv4.ip_forward)" + echo "hash=$(sysctl -n net.ipv4.fib_multipath_hash_policy)" + echo "rpf=$(sysctl -n net.ipv4.conf.all.rp_filter)" + register: vpn_sysctl_live + changed_when: false + tags: [setup, verify] -# ========================================================================== -# GRE-over-FOU mode (Crusoe): GRE encapsulated in UDP so it passes through -# firewalls that block raw GRE (IP protocol 47). -# ========================================================================== +- name: Assert the critical sysctls actually took effect + ansible.builtin.assert: + that: + - "'forward=1' in vpn_sysctl_live.stdout" + - "'hash=' ~ (vpn_multipath_hash_policy | string) in vpn_sysctl_live.stdout" + - "'rpf=' ~ (vpn_rp_filter | string) in vpn_sysctl_live.stdout" + fail_msg: >- + A sysctl this design depends on did not take effect. Live values: + {{ vpn_sysctl_live.stdout_lines }}. Something applied later is + overriding /etc/sysctl.d/99-vpn-gateway.conf - check /etc/sysctl.conf, + which "sysctl --system" applies last. + tags: [setup, verify] -- name: Load FOU kernel module - community.general.modprobe: - name: fou - state: present - when: vpn_use_gre | default(false) - tags: [setup] +# --------------------------------------------------------------------------- +# Network setup - one rendered script, used by the play AND by systemd +# --------------------------------------------------------------------------- -- name: Create GRE-over-FOU tunnel - ansible.builtin.shell: | - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true - sleep 1 - ip fou add port {{ vpn_fou_port }} ipproto 47 - ip link add name gre-vpn type gre local {{ vpn_local_gw_ip }} key {{ vpn_gre_key }} ttl 255 \ - encap fou encap-sport {{ vpn_fou_port }} encap-dport {{ vpn_fou_port }} - ip link set gre-vpn mtu {{ vpn_gre_mtu }} - ip link set gre-vpn up - changed_when: true - when: vpn_use_gre | default(false) +- name: Install host tuning script + ansible.builtin.template: + src: perf-tuning.sh.j2 + dest: /usr/local/sbin/vpn-perf-tuning.sh + mode: "0755" + when: vpn_perf_tuning | bool tags: [setup] -- name: Populate GRE neighbor table from client CIDR +# --------------------------------------------------------------------------- +# Kernel AES-GCM implementation +# --------------------------------------------------------------------------- +# The kernel binds the highest-priority driver registered for +# rfc4106(gcm(aes)). Kernels before 6.11 ship only the 2010-era AES-NI code. +# +# Measured: one tunnel, same two VMs, same WAN path (9.01 Gbps raw control). +# 6.8.0-78 rfc4106-gcm-aesni 4.67 Gbps mean decrypt core 83% +# 6.11.0-29 rfc4106-gcm-vaes-avx10_512 5.75 Gbps mean decrypt core 62% +# +# A kernel change needs a reboot, so this reports rather than acts. +- name: Detect the kernel AES-GCM implementation ansible.builtin.shell: | - python3 -c " - import ipaddress, subprocess, sys - gw = '{{ vpn_local_gw_ip }}' - net = ipaddress.ip_network('{{ vpn_client_cidr }}', strict=False) - for ip in net.hosts(): - s = str(ip) - if s == gw: - continue - subprocess.run( - ['ip', 'neigh', 'replace', s, 'lladdr', s, 'dev', 'gre-vpn', 'nud', 'permanent'], - check=True - ) - print(f'Populated {net.num_addresses - 2} neighbor entries for {net}') - " - changed_when: true - when: vpn_use_gre | default(false) - tags: [setup] + modprobe aesni_intel 2>/dev/null || true + # /proc/crypto also lists internal instances ("__rfc4106(...)") and + # wrappers ("seqiv(rfc4106(...))"). Anchoring the name selects the entry + # consumers actually bind. sed BRE treats "(" literally, so no escaping. + drv=$(sed -n "/^name .*: rfc4106(gcm(aes))$/,+1p" /proc/crypto \ + | sed -n "s/^driver *: *//p" | head -1) + vaes=no; grep -qw vaes /proc/cpuinfo && vaes=yes + echo "${drv:-unknown}|${vaes}|$(uname -r)" + register: vpn_gcm_probe + changed_when: false + failed_when: false + when: vpn_check_crypto_accel | bool + tags: [setup, verify] -- name: Add ip rule for xfrm0 → GRE policy routing - ansible.builtin.shell: | - ip rule del iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }} 2>/dev/null || true - ip rule add iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }} - changed_when: true - when: vpn_use_gre | default(false) - tags: [setup] +- name: Parse the AES-GCM probe + ansible.builtin.set_fact: + vpn_gcm_driver: "{{ (vpn_gcm_probe.stdout | trim).split('|')[0] }}" + vpn_cpu_has_vaes: "{{ (vpn_gcm_probe.stdout | trim).split('|')[1] == 'yes' }}" + vpn_running_kernel: "{{ (vpn_gcm_probe.stdout | trim).split('|')[2] }}" + when: + - vpn_check_crypto_accel | bool + - vpn_gcm_probe.stdout is defined + - "'|' in (vpn_gcm_probe.stdout | default(''))" + tags: [setup, verify] -- name: Route client CIDR through GRE in policy table - ansible.builtin.command: > - ip route replace {{ vpn_client_cidr }} dev gre-vpn table {{ vpn_rt_table }} - changed_when: true - when: vpn_use_gre | default(false) - tags: [setup] +- name: Report the AES-GCM implementation + ansible.builtin.debug: + msg: >- + AES-GCM driver={{ vpn_gcm_driver }} kernel={{ vpn_running_kernel }} + {% if 'vaes' in vpn_gcm_driver %} + - VAES fast path, nothing to do. + {% elif vpn_cpu_has_vaes | bool %} + - SLOW PATH. This CPU supports VAES but kernel {{ vpn_running_kernel }} + has no VAES AES-GCM driver (added in 6.11). Measured +23% per tunnel + after upgrading. Install linux-image-generic-6.11 (or 6.14) and reboot; + see AZURE-10G-GUIDE.md. Do NOT jump to 7.0 without measuring - it + regressed 2.4x on this hardware. + {% else %} + - this CPU has no VAES, so the AES-NI driver is the best available. + {% endif %} + when: + - vpn_check_crypto_accel | bool + - vpn_gcm_driver is defined + tags: [setup, verify] -# --- Mark-based routing: GRE traffic → xfrm0 --- -# For full tunnel (vpn_remote_subnet = 0.0.0.0/0), traffic arriving on -# gre-vpn may be destined for any IP (not just the datacenter CIDR). -# We mark it and route via a separate table through xfrm0, so the -# gateway's own default route is unaffected. +- name: Require VAES AES-GCM + ansible.builtin.assert: + that: "'vaes' in vpn_gcm_driver" + fail_msg: >- + vpn_require_crypto_accel=true but the kernel AES-GCM driver is + {{ vpn_gcm_driver }} on kernel {{ vpn_running_kernel }}. Install + linux-image-generic-6.11 (or 6.14) and reboot, or set + vpn_require_crypto_accel=false to proceed on the slow path. + when: + - vpn_require_crypto_accel | bool + - vpn_gcm_driver is defined + tags: [setup, verify] -- name: Mark GRE-inbound traffic for IPsec - ansible.builtin.shell: | - iptables -t mangle -D PREROUTING -i gre-vpn -j MARK --set-mark 0x64 2>/dev/null || true - iptables -t mangle -A PREROUTING -i gre-vpn -j MARK --set-mark 0x64 - changed_when: true - when: vpn_use_gre | default(false) +- name: Install vpn-network script + ansible.builtin.template: + src: vpn-network.sh.j2 + dest: /usr/local/sbin/vpn-network.sh + mode: "0755" tags: [setup] -- name: Add ip rule for marked traffic → xfrm0 - ansible.builtin.shell: | - ip rule del fwmark 0x64 lookup 200 2>/dev/null || true - ip rule add fwmark 0x64 lookup 200 +- name: Bring up XFRM interfaces, routes, marks + ansible.builtin.command: /usr/local/sbin/vpn-network.sh up + register: vpn_network_up changed_when: true - when: vpn_use_gre | default(false) tags: [setup] -- name: Route all traffic through xfrm0 in mark table - ansible.builtin.command: > - ip route replace default dev {{ vpn_xfrm_iface }} table 200 - changed_when: true - when: vpn_use_gre | default(false) +- name: Show vpn-network output + ansible.builtin.debug: + var: vpn_network_up.stdout_lines tags: [setup] -# ========================================================================== +# --------------------------------------------------------------------------- # Firewall -# ========================================================================== +# +# Interface wildcards ("xfrm+", "gre-vpn+") keep the rule count flat no matter +# how many tunnels or FOU ports are configured. +# --------------------------------------------------------------------------- - name: Flush FORWARD chain ansible.builtin.iptables: @@ -193,50 +496,79 @@ policy: DROP tags: [setup] -# --- GRE mode: forward between GRE and XFRM --- -- name: "Allow forwarding: GRE → XFRM" +- name: "Allow forwarding: client overlay <-> IPsec" ansible.builtin.iptables: chain: FORWARD - in_interface: "gre-vpn" - out_interface: "{{ vpn_xfrm_iface }}" + in_interface: "{{ item.i }}" + out_interface: "{{ item.o }}" jump: ACCEPT - comment: "VPN: client GRE -> IPsec" - when: vpn_use_gre | default(false) + comment: "{{ item.c }}" + loop: + - { i: "gre-vpn+", o: "xfrm+", c: "VPN: client GRE -> IPsec" } + - { i: "xfrm+", o: "gre-vpn+", c: "VPN: IPsec -> client GRE" } + when: vpn_transport == 'gre_fou' tags: [setup] -- name: "Allow forwarding: XFRM → GRE" +# Covers route transport in both directions, and internet egress for +# full-tunnel mode, without needing per-tunnel rules. +- name: "Allow forwarding: LAN <-> IPsec" ansible.builtin.iptables: chain: FORWARD - in_interface: "{{ vpn_xfrm_iface }}" - out_interface: "gre-vpn" + in_interface: "{{ item.i }}" + out_interface: "{{ item.o }}" jump: ACCEPT - comment: "VPN: IPsec -> client GRE" - when: vpn_use_gre | default(false) + comment: "{{ item.c }}" + loop: + - { i: "xfrm+", o: "{{ vpn_primary_iface }}", c: "VPN: IPsec -> LAN/internet" } + - { i: "{{ vpn_primary_iface }}", o: "xfrm+", c: "VPN: LAN/internet -> IPsec" } tags: [setup] -# --- Internet egress: forward decrypted traffic to LAN/internet --- -- name: "Allow forwarding: XFRM → LAN (internet egress)" +# Narrower duplicates of the pair above, kept for the direct-forwarding mode +# this role has always shipped. +- name: "Allow forwarding: local subnet <-> IPsec (direct mode)" ansible.builtin.iptables: chain: FORWARD - in_interface: "{{ vpn_xfrm_iface }}" - out_interface: "{{ vpn_primary_iface }}" + in_interface: "{{ item.i }}" + out_interface: "{{ item.o }}" + source: "{{ item.s }}" + destination: "{{ item.d }}" jump: ACCEPT - comment: "VPN: IPsec -> internet" - tags: [setup] - -- name: "Allow forwarding: LAN → XFRM (internet return)" + comment: "{{ item.c }}" + loop: + - { i: "{{ vpn_primary_iface }}", o: "xfrm+", s: "{{ vpn_local_subnet }}", + d: "{{ vpn_remote_subnet }}", c: "VPN: local subnet -> IPsec" } + - { i: "xfrm+", o: "{{ vpn_primary_iface }}", s: "{{ vpn_remote_subnet }}", + d: "{{ vpn_local_subnet }}", c: "VPN: IPsec -> local subnet" } + when: vpn_transport == 'none' + tags: [setup] + +# The internet-egress SNAT below matches "source = remote subnet, out = uplink". +# In route transport, traffic heading BACK to local clients also leaves via the +# uplink with a remote source address, so it would match and be NAT'd - which +# both hides the real remote IPs and makes the path stateful, breaking +# multi-gateway ECMP. This RETURN exempts client-bound traffic, and must be +# added BEFORE the SNAT rule. +- name: "Exempt client-bound traffic from the internet SNAT" ansible.builtin.iptables: - chain: FORWARD - in_interface: "{{ vpn_primary_iface }}" - out_interface: "{{ vpn_xfrm_iface }}" - jump: ACCEPT - comment: "VPN: internet -> IPsec" + table: nat + chain: POSTROUTING + out_interface: "{{ vpn_primary_iface }}" + source: "{{ vpn_remote_subnet }}" + destination: "{{ vpn_client_cidr }}" + jump: RETURN + comment: "VPN: never SNAT traffic destined for local clients" + when: + - vpn_snat_internet_egress | bool + - vpn_serves_clients | bool tags: [setup] -# --- SNAT for internet-bound traffic from VPN clients --- -# MASQUERADE doesn't work here because XFRM bypasses conntrack for -# forwarded packets. Explicit SNAT with source match is reliable. -- name: "SNAT decrypted VPN traffic going to internet" +# MASQUERADE does not work here because XFRM bypasses conntrack for forwarded +# packets. Explicit SNAT with a source match is reliable. +# +# NOTE: SNAT is STATEFUL. A deployment with several gateway VMs sharing the +# ECMP load cannot use it, because the reply may return through a different +# gateway which has no matching conntrack entry. +- name: "SNAT decrypted VPN traffic going to the internet" ansible.builtin.iptables: table: nat chain: POSTROUTING @@ -245,67 +577,73 @@ jump: SNAT to_source: "{{ vpn_local_gw_ip }}" comment: "VPN: SNAT for internet egress" + when: vpn_snat_internet_egress | bool tags: [setup] -# --- Direct mode (Azure/GCP/AWS): forward between LAN and XFRM --- -- name: "Allow forwarding: LAN → XFRM" +- name: "SNAT remote traffic towards clients (route transport fallback)" ansible.builtin.iptables: - chain: FORWARD - in_interface: "{{ vpn_primary_iface }}" - out_interface: "{{ vpn_xfrm_iface }}" - source: "{{ vpn_local_subnet }}" - destination: "{{ vpn_remote_subnet }}" - jump: ACCEPT - comment: "VPN: local subnet -> IPsec" - when: not (vpn_use_gre | default(false)) - tags: [setup] - -- name: "Allow forwarding: XFRM → LAN" - ansible.builtin.iptables: - chain: FORWARD - in_interface: "{{ vpn_xfrm_iface }}" + table: nat + chain: POSTROUTING out_interface: "{{ vpn_primary_iface }}" source: "{{ vpn_remote_subnet }}" - destination: "{{ vpn_local_subnet }}" - jump: ACCEPT - comment: "VPN: IPsec -> local subnet" - when: not (vpn_use_gre | default(false)) + destination: "{{ vpn_client_cidr }}" + jump: SNAT + to_source: "{{ vpn_local_gw_ip }}" + comment: "VPN: SNAT return traffic past port security" + when: vpn_transport == 'route' and vpn_route_return == 'snat' tags: [setup] -# --- Common rules --- - name: Allow established/related forwarding ansible.builtin.iptables: chain: FORWARD ctstate: [ESTABLISHED, RELATED] jump: ACCEPT + when: vpn_stateful_forward_rule | bool tags: [setup] -- name: Allow IKE and ESP from remote gateway +- name: Allow IKE and ESP from every remote gateway ansible.builtin.iptables: chain: INPUT - protocol: "{{ item.proto }}" - destination_port: "{{ item.port | default(omit) }}" - source: "{{ vpn_remote_gw_ip }}" + protocol: "{{ item.1.proto }}" + destination_port: "{{ item.1.port | default(omit) }}" + source: "{{ item.0 }}" jump: ACCEPT - loop: - - { proto: udp, port: "500" } - - { proto: udp, port: "4500" } - - { proto: esp } + comment: "VPN: IKE/ESP from {{ item.0 }}" + loop: "{{ (vpn_tunnels | map(attribute='remote_addr') | unique | list) | product(vpn_ike_ports) | list }}" + vars: + vpn_ike_ports: + - { proto: udp, port: "500" } + - { proto: udp, port: "4500" } + - { proto: esp } tags: [setup] -- name: Allow FOU (UDP {{ vpn_fou_port }}) from client CIDR +- name: Allow FOU from the client CIDR ansible.builtin.iptables: chain: INPUT protocol: udp - destination_port: "{{ vpn_fou_port }}" + destination_port: "{{ (vpn_fou_port | int) + item }}" source: "{{ vpn_client_cidr }}" jump: ACCEPT - when: vpn_use_gre | default(false) + comment: "VPN: GRE-over-FOU from clients" + loop: "{{ range(vpn_fou_port_count | int) | list }}" + when: vpn_transport == 'gre_fou' + tags: [setup] + +- name: Allow BGP from the peer addresses + ansible.builtin.iptables: + chain: INPUT + protocol: tcp + destination_port: "179" + source: "{{ item }}" + jump: ACCEPT + comment: "VPN: BGP from {{ item }}" + loop: "{{ vpn_tunnels | map(attribute='bgp_peer') | reject('equalto', '') | unique | list }}" + when: vpn_bgp_enabled | bool tags: [setup] -# ========================================================================== +# --------------------------------------------------------------------------- # strongSwan -# ========================================================================== +# --------------------------------------------------------------------------- - name: Ensure swanctl conf directory exists ansible.builtin.file: @@ -314,6 +652,15 @@ mode: "0755" tags: [setup] +- name: Configure charon + ansible.builtin.template: + src: strongswan-vpn.conf.j2 + dest: /etc/strongswan.d/99-vpn.conf + mode: "0644" + notify: restart strongswan + failed_when: false + tags: [setup] + - name: Write swanctl configuration ansible.builtin.template: src: swanctl-vpn.conf.j2 @@ -331,21 +678,80 @@ failed_when: false tags: [setup] +- name: Flush handlers so strongSwan reloads before verification + ansible.builtin.meta: flush_handlers + tags: [setup] + - name: Load swanctl configuration ansible.builtin.command: swanctl --load-all changed_when: false failed_when: false tags: [setup] -# ========================================================================== +- name: Check strongSwan can do AES-GCM + ansible.builtin.shell: >- + swanctl --list-algs 2>/dev/null | grep -qi gcm + || ipsec listalgs 2>/dev/null | grep -qi gcm + register: vpn_gcm_check + changed_when: false + failed_when: false + tags: [setup, verify] + +- name: "WARNING: no AES-GCM in strongSwan" + ansible.builtin.debug: + msg: >- + Could not confirm AES-GCM support in strongSwan. The gcmaes256 crypto + profiles will fail to negotiate. Install libstrongswan-standard-plugins + (Debian/Ubuntu) or the equivalent gcm/openssl plugin. + when: + - vpn_assert_gcm_plugin | bool + - vpn_gcm_check.rc != 0 + tags: [setup, verify] + +# --------------------------------------------------------------------------- +# BGP (FRR) +# --------------------------------------------------------------------------- + +- name: Enable only zebra and bgpd + ansible.builtin.template: + src: frr-daemons.j2 + dest: /etc/frr/daemons + mode: "0640" + owner: frr + group: frr + when: vpn_bgp_enabled | bool + notify: restart frr + tags: [setup] + +- name: Write FRR configuration + ansible.builtin.template: + src: frr.conf.j2 + dest: /etc/frr/frr.conf + mode: "0640" + owner: frr + group: frr + when: vpn_bgp_enabled | bool + notify: restart frr + tags: [setup] + +- name: Enable and start FRR + ansible.builtin.systemd: + name: frr + state: started + enabled: true + when: vpn_bgp_enabled | bool + tags: [setup] + +# --------------------------------------------------------------------------- # Persistence -# ========================================================================== +# --------------------------------------------------------------------------- - name: Write vpn-network systemd unit ansible.builtin.template: src: vpn-network.service.j2 dest: /etc/systemd/system/vpn-network.service mode: "0644" + notify: reload systemd tags: [setup] - name: Enable vpn-network service @@ -355,95 +761,199 @@ daemon_reload: true tags: [setup] -# ========================================================================== +- name: Install ECMP health script + ansible.builtin.template: + src: ecmp-health.sh.j2 + dest: /usr/local/sbin/vpn-ecmp-health.sh + mode: "0755" + when: vpn_ecmp_healthcheck_effective | bool + tags: [setup] + +- name: Write ECMP health unit and timer + ansible.builtin.template: + src: "ecmp-health.{{ item }}.j2" + dest: "/etc/systemd/system/vpn-ecmp-health.{{ item }}" + mode: "0644" + loop: [service, timer] + when: vpn_ecmp_healthcheck_effective | bool + notify: reload systemd + tags: [setup] + +- name: Enable ECMP health timer + ansible.builtin.systemd: + name: vpn-ecmp-health.timer + state: started + enabled: true + daemon_reload: true + when: vpn_ecmp_healthcheck_effective | bool + tags: [setup] + +# --------------------------------------------------------------------------- # Verify -# ========================================================================== +# --------------------------------------------------------------------------- + +- name: Verify every XFRM interface is up + ansible.builtin.shell: >- + ip link show {{ item.xfrm_iface }} | grep -qE 'UP|UNKNOWN' + loop: "{{ vpn_tunnels }}" + loop_control: + label: "{{ item.xfrm_iface }}" + changed_when: false + tags: [setup, verify] + +- name: Verify GRO is off on the uplink (GRE-over-FOU correctness) + ansible.builtin.shell: >- + ethtool -k {{ vpn_primary_iface }} | awk '/generic-receive-offload/{print $2}' + register: vpn_gro_state + changed_when: false + failed_when: false + when: vpn_transport == 'gre_fou' and (vpn_fou_disable_gro | bool) + tags: [setup, verify] + +- name: "WARNING: GRO is still on and will destroy TCP" + ansible.builtin.debug: + msg: >- + GRO is still ON for {{ vpn_primary_iface }}. It coalesces inbound + FOU/UDP packets that then cannot be re-encapsulated, so they are + dropped. UDP and ICMP keep working while TCP collapses - measured + 3.65 Mbps instead of 4590 Mbps. Run + "ethtool -K {{ vpn_primary_iface }} gro off" and find out what turned + it back on (a NetworkManager or netplan hook is the usual culprit). + when: + - vpn_transport == 'gre_fou' + - vpn_fou_disable_gro | bool + - (vpn_gro_state.stdout | default('') | trim) == 'on' + tags: [setup, verify] + +- name: Verify the ECMP hash policy took + ansible.builtin.command: sysctl -n net.ipv4.fib_multipath_hash_policy + register: vpn_hash_policy + changed_when: false + tags: [setup, verify] + +- name: "WARNING: ECMP hash policy is L3-only" + ansible.builtin.debug: + msg: >- + net.ipv4.fib_multipath_hash_policy is + {{ vpn_hash_policy.stdout | trim }}, not 1. Every flow between the same + pair of hosts will take ONE tunnel and the others will sit idle. The + setup will look correct and perform like a single tunnel. + when: + - (vpn_tunnels | length) > 1 + - vpn_hash_policy.stdout | trim != '1' + tags: [setup, verify] + +- name: Count established SAs + ansible.builtin.shell: swanctl --list-sas 2>/dev/null | grep -c ESTABLISHED + register: vpn_sa_count + changed_when: false + failed_when: false + tags: [setup, verify] + +- name: "WARNING: fewer established SAs than tunnels" + ansible.builtin.debug: + msg: >- + Only {{ vpn_sa_count.stdout | default('0') | trim }} of + {{ vpn_tunnels | length }} tunnel(s) are ESTABLISHED. With + start_action={{ vpn_start_action_effective }} they should all come up at + deploy time. A tunnel that is down still has an UP xfrm interface, so it + keeps attracting its share of ECMP traffic and blackholes it. Check + "journalctl -u strongswan" and that UDP 500/4500 reaches + {{ vpn_tunnels | map(attribute='remote_addr') | unique | join(', ') }}. + when: (vpn_sa_count.stdout | default('0') | trim | int) < (vpn_tunnels | length) + tags: [setup, verify] -- name: Verify XFRM interface is up - ansible.builtin.shell: ip link show {{ vpn_xfrm_iface }} | grep -qE 'UP|UNKNOWN' +- name: Check the negotiated ESP proposal is AEAD + ansible.builtin.shell: swanctl --list-sas 2>/dev/null | grep -oiE 'AES_GCM[_A-Z0-9]*' | head -1 + register: vpn_negotiated_esp changed_when: false + failed_when: false + when: vpn_assert_negotiated_esp | bool + tags: [setup, verify] + +- name: "WARNING: live SAs are not using AES-GCM" + ansible.builtin.debug: + msg: >- + No AES-GCM found in the live SAs. Azure's own numbers are 2.3 Gbps per + tunnel with GCMAES256 versus 700 Mbps with AES256+SHA256, so this is + worth roughly 3x. Set vpn_crypto_profile=gcmaes256 here AND a matching + custom IPsec/IKE policy on the peer connection. + when: + - vpn_assert_negotiated_esp | bool + - (vpn_sa_count.stdout | default('0') | int) > 0 + - (vpn_negotiated_esp.stdout | default('')) == '' tags: [setup, verify] -- name: Count GRE neighbor entries (GRE mode) - ansible.builtin.shell: ip neigh show dev gre-vpn nud permanent | wc -l - register: neigh_count +- name: Check BGP sessions + ansible.builtin.command: vtysh -c "show ip bgp summary" + register: vpn_bgp_summary changed_when: false - when: vpn_use_gre | default(false) + failed_when: false + when: vpn_bgp_enabled | bool tags: [setup, verify] - name: Report status ansible.builtin.debug: msg: >- - Gateway {{ inventory_hostname }} ready - (mode={{ 'GRE' if vpn_use_gre | default(false) else 'direct' }}) - {% if vpn_use_gre | default(false) %} - — {{ neigh_count.stdout | default('0') | trim }} GRE neighbors for {{ vpn_client_cidr }} - {% endif %} - — IPsec peer {{ vpn_remote_gw_ip }} - — protecting {{ vpn_local_subnet }} ↔ {{ vpn_remote_subnet }} + Gateway {{ inventory_hostname }}: + {{ vpn_tunnels | length }} tunnel(s) + [{{ vpn_tunnels | map(attribute='xfrm_iface') | join(', ') }}] + -> {{ vpn_tunnels | map(attribute='remote_addr') | unique | join(', ') }} + | transport={{ vpn_transport }} + | crypto={{ vpn_crypto_profile }} (esp={{ vpn_esp_proposals }}) + | established SAs={{ vpn_sa_count.stdout | default('?') | trim }} + | bgp={{ 'on' if vpn_bgp_enabled | bool else 'off' }} + | perf_tuning={{ vpn_perf_tuning | bool }} + | protecting {{ vpn_child_local_ts }} <-> {{ vpn_child_remote_ts }} tags: [setup, verify] -# ========================================================================== +# --------------------------------------------------------------------------- # Teardown -# ========================================================================== +# --------------------------------------------------------------------------- - name: "Teardown: stop services" ansible.builtin.systemd: name: "{{ item }}" state: stopped enabled: false - loop: [vpn-network, strongswan] + loop: [vpn-ecmp-health.timer, vpn-network, frr, strongswan] failed_when: false tags: [teardown, never] -- name: "Teardown: remove systemd unit" - ansible.builtin.file: - path: /etc/systemd/system/vpn-network.service - state: absent - tags: [teardown, never] - -- name: "Teardown: remove ip rule (GRE mode)" - ansible.builtin.command: ip rule del iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }} - failed_when: false - changed_when: true - when: vpn_use_gre | default(false) - tags: [teardown, never] - -- name: "Teardown: flush policy table (GRE mode)" - ansible.builtin.command: ip route flush table {{ vpn_rt_table }} +- name: "Teardown: tear down the network" + ansible.builtin.command: /usr/local/sbin/vpn-network.sh down failed_when: false changed_when: true - when: vpn_use_gre | default(false) tags: [teardown, never] -- name: "Teardown: remove GRE tunnel and FOU port" - ansible.builtin.shell: | - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true - failed_when: false - changed_when: true - when: vpn_use_gre | default(false) - tags: [teardown, never] - -- name: "Teardown: remove XFRM interface" - ansible.builtin.command: ip link del {{ vpn_xfrm_iface }} - failed_when: false - changed_when: true - tags: [teardown, never] - -- name: "Teardown: remove configs" +- name: "Teardown: remove units, scripts and configs" ansible.builtin.file: path: "{{ item }}" state: absent loop: + - /etc/systemd/system/vpn-network.service + - /etc/systemd/system/vpn-ecmp-health.service + - /etc/systemd/system/vpn-ecmp-health.timer + - /usr/local/sbin/vpn-network.sh + - /usr/local/sbin/vpn-ecmp-health.sh + - /usr/local/sbin/vpn-perf-tuning.sh - /etc/swanctl/conf.d/site-vpn.conf - /etc/sysctl.d/99-vpn-gateway.conf + - /etc/strongswan.d/99-vpn.conf tags: [teardown, never] - name: "Teardown: reset iptables" ansible.builtin.shell: | iptables -F FORWARD iptables -P FORWARD ACCEPT + iptables -t mangle -F PREROUTING + iptables -t mangle -F FORWARD + iptables -t raw -F PREROUTING changed_when: true + failed_when: false + tags: [teardown, never] + +- name: "Teardown: reload systemd" + ansible.builtin.systemd: + daemon_reload: true tags: [teardown, never] diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/preflight.yml b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/preflight.yml new file mode 100644 index 0000000..3730bc1 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/tasks/preflight.yml @@ -0,0 +1,411 @@ +--- +# ============================================================================= +# Preflight - runs as the FIRST play in site.yml, with any_errors_fatal. +# +# Nothing is configured until every check here passes on every host. This +# replaces the old scripts/preflight.sh: same checks, but it cannot be +# forgotten, it reads the real inventory instead of two hand-typed SSH +# targets, and it covers the client VMs too. +# +# Four groups of checks: +# 1. configuration are the five required values actually filled in +# 2. host readiness can this machine run the role at all +# 3. overlay sanity do the client VMs fall inside vpn_client_cidr +# 4. reachability does UDP 500/4500 pass BOTH ways between gateways +# +# Check 4 is the one that saves the most time. IPsec needs both ports in both +# directions, and a one-way block looks exactly like a broken config: IKE +# times out and the log says nothing useful. It can only run between gateways +# that are hosts in this inventory AND in this run - a managed cloud peer has +# nothing to run a listener on, and a peer excluded by --limit cannot start +# one either. +# ============================================================================= + +- name: "Preflight: derive roles for this host" + ansible.builtin.set_fact: + vpn_pf_is_gateway: "{{ inventory_hostname in (groups['vpn_gateways'] | default([])) }}" + vpn_pf_is_client: "{{ inventory_hostname in (groups['vpn_clients'] | default([])) }}" + tags: [preflight, always] + +# --------------------------------------------------------------------------- +# 1. Configuration +# --------------------------------------------------------------------------- +# The "<" test catches every "" placeholder the examples +# ship with, which is the failure a first-time deploy actually hits. +- name: "Preflight: the values only you can know" + ansible.builtin.assert: + that: + - vpn_psk is defined and (vpn_psk | length) > 8 and '<' not in vpn_psk + # Counting addresses is not enough - the shipped example has TWO and + # both are placeholders. + - (vpn_remote_addrs | default([]) | select('search', '<') | list | length) == 0 + - (vpn_remote_addrs | default([]) | length) > 0 + or (vpn_remote_gw_ip is defined and '<' not in (vpn_remote_gw_ip | string)) + - vpn_local_subnet is defined and '<' not in (vpn_local_subnet | string) + - vpn_remote_subnet is defined and '<' not in (vpn_remote_subnet | string) + - vpn_client_cidr is defined and '<' not in (vpn_client_cidr | string) + fail_msg: >- + group_vars/all.yml still has placeholders. Five values are required: + vpn_psk (openssl rand -base64 48, and the same string on the peer), + vpn_remote_addrs (the peer's public IPs - list BOTH of an Azure + active-active gateway's addresses to get two tunnels), vpn_local_subnet + (your Crusoe CIDR), vpn_remote_subnet (the peer's CIDR) and + vpn_client_cidr (which Crusoe IPs join the overlay). + If you set these in your OWN inventory's group_vars/all.yml, note that + this playbook's group_vars/all.yml outranks it - see the precedence note + at the top of that file. + success_msg: "Configuration is complete." + when: vpn_pf_is_gateway | bool + tags: [preflight, always] + +# The IKE identity a cloud peer matches on is this gateway's PUBLIC IP, and it +# is derived from ansible_host. A DNS name there would become an FQDN +# identity that Azure/AWS/GCP silently fail to match. Only checked when the +# identity was not overridden by hand. +- name: "Preflight: the gateway's ansible_host is an IP address, not a name" + ansible.builtin.assert: + that: vpn_local_outer_ip is match('^([0-9]{1,3}\.){3}[0-9]{1,3}$') + fail_msg: >- + {{ inventory_hostname }} has ansible_host={{ vpn_local_outer_ip }}, + which is not an IPv4 address. The IKE local identity is derived from it, + and a managed cloud peer identifies this device by its public IP, so the + tunnel would never match. Put the gateway's public IP in inventory.ini, + or set vpn_local_id explicitly. + when: + - vpn_pf_is_gateway | bool + - vpn_local_id == vpn_local_outer_ip + tags: [preflight, always] + +# --------------------------------------------------------------------------- +# 2. Host readiness +# --------------------------------------------------------------------------- +# First contact with the hosts. Everything above is controller-side, so a +# fresh clone with placeholder IPs fails with the message above rather than +# an SSH timeout. +- name: "Preflight: gather facts" + ansible.builtin.setup: + tags: [preflight, always] + +- name: "Preflight: probe host readiness" + ansible.builtin.shell: + cmd: | + set -u + aes=no; grep -qw aes /proc/cpuinfo && aes=yes + vaes=no; grep -qw vaes /proc/cpuinfo && vaes=yes + # Modules the datapath needs. Built-in counts, so check /proc/modules + # and the module database, not just the loaded list. + miss="" + for m in xfrm_interface fou ip_gre esp4; do + modinfo "$m" >/dev/null 2>&1 && continue + grep -qw "$m" /proc/modules 2>/dev/null && continue + miss="$miss $m" + done + # strongSwan and the kernel both come from apt, so unreachable mirrors + # are a hard stop on a GATEWAY - and a much clearer one here than + # mid-install. A fresh VM often holds the apt lock for a minute + # (unattended-upgrades), so retry rather than misreport that as + # "unreachable". + apt=unreachable + n=0 + while [ "$n" -lt 3 ]; do + if timeout 60 apt-get -qq update >/dev/null 2>&1; then apt=ok; break; fi + if fuser /var/lib/apt/lists/lock /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then + apt=locked + else + apt=unreachable + fi + n=$((n + 1)); sleep 10 + done + printf '{"aes":"%s","vaes":"%s","missing":"%s","apt":"%s"}\n' \ + "$aes" "$vaes" "${miss# }" "$apt" + register: vpn_pf_host + changed_when: false + check_mode: false + tags: [preflight, always] + +- name: "Preflight: record host readiness" + ansible.builtin.set_fact: + vpn_pf: "{{ vpn_pf_host.stdout | trim | from_json }}" + tags: [preflight, always] + +- name: "Preflight: this machine can run the role" + ansible.builtin.assert: + that: + - ansible_facts.os_family == 'Debian' + - (not (vpn_pf_is_gateway | bool)) or vpn_pf.apt == 'ok' + - vpn_pf.missing | length == 0 + - (ansible_facts[ansible_facts.default_ipv4.interface].mtu | int) >= 1500 + fail_msg: >- + {{ inventory_hostname }} is not ready. + os_family={{ ansible_facts.os_family }} (this role installs with apt, so + Debian or Ubuntu is required). + apt={{ vpn_pf.apt }} (gateways install strongSwan and the kernel from + apt; "locked" means another apt process held the lock for the whole + retry window - wait for unattended-upgrades to finish and re-run). + missing kernel modules='{{ vpn_pf.missing }}' (the datapath needs + xfrm_interface, fou, ip_gre and esp4). + uplink {{ ansible_facts.default_ipv4.interface }} + mtu={{ ansible_facts[ansible_facts.default_ipv4.interface].mtu }} - below + 1500 the GRE overlay will not fit, so lower vpn_gre_mtu and vpn_xfrm_mtu + to match, or use a network with a normal MTU. + success_msg: >- + {{ inventory_hostname }} ready: {{ ansible_facts.distribution }} + {{ ansible_facts.distribution_version }}, uplink + {{ ansible_facts.default_ipv4.interface }} mtu + {{ ansible_facts[ansible_facts.default_ipv4.interface].mtu }}, + aes-ni={{ vpn_pf.aes }}, apt={{ vpn_pf.apt }} + tags: [preflight, always] + +# Clients install nothing, so apt trouble there is worth a line, not a stop. +- name: "Preflight: note apt trouble on a client" + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: apt={{ vpn_pf.apt }}. The client role installs + no packages, so this does not block the deployment - but the host will + not be able to install anything else until it is resolved. + when: + - vpn_pf_is_client | bool + - not (vpn_pf_is_gateway | bool) + - vpn_pf.apt != 'ok' + tags: [preflight, always] + +# AES-NI is not strictly required, but without it GCM is roughly an order of +# magnitude slower, which is worth saying out loud rather than discovering +# from a throughput test. +- name: "Preflight: warn when the CPU has no AES-NI" + ansible.builtin.debug: + msg: >- + WARNING: {{ inventory_hostname }} reports no AES-NI. AES-GCM will be + roughly 10x slower in software. Check the instance type. + when: vpn_pf.aes != 'yes' + tags: [preflight, always] + +- name: "Preflight: note the crypto acceleration on offer" + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }} has VAES, so the kernel play will move it to + {{ vpn_kernel_package }} if the running kernel predates 6.11. Measured + +23% per tunnel. On a host that already carries a VPN the kernel is + installed but the reboot is left to you (vpn_kernel_reboot_live). + when: + - vpn_pf_is_gateway | bool + - vpn_pf.vaes == 'yes' + - vpn_kernel_upgrade | bool + tags: [preflight, always] + +# --------------------------------------------------------------------------- +# 3. Overlay sanity +# --------------------------------------------------------------------------- +# A client VM outside every gateway's vpn_client_cidr gets no GRE neighbour +# entry and no mark rule, so its traffic silently never enters the tunnel. +# Cheap to check here, very annoying to diagnose later. +- name: "Preflight: collect every gateway's client CIDR" + ansible.builtin.set_fact: + vpn_pf_cidrs: >- + {{ groups['vpn_gateways'] | default([]) + | map('extract', hostvars, 'vpn_client_cidr') + | select('defined') | reject('equalto', None) | list }} + tags: [preflight, always] + +- name: "Preflight: this client is inside the overlay" + ansible.builtin.shell: + cmd: | + python3 - <<'PYEOF' + import ipaddress as ipa + me = ipa.ip_address("{{ ansible_facts.default_ipv4.address }}") + nets = {{ vpn_pf_cidrs | to_json }} + hits = [n for n in nets if me in ipa.ip_network(n, strict=False)] + print("IN " + hits[0] if hits else "OUT") + PYEOF + register: vpn_pf_inside + changed_when: false + check_mode: false + when: + - vpn_pf_is_client | bool + - vpn_pf_cidrs | length > 0 + tags: [preflight, always] + +- name: "Preflight: refuse a client outside every client CIDR" + ansible.builtin.assert: + that: vpn_pf_inside.stdout is search('^IN ') + fail_msg: >- + {{ inventory_hostname }} has private IP + {{ ansible_facts.default_ipv4.address }}, which is outside every + gateway's vpn_client_cidr ({{ vpn_pf_cidrs | join(', ') }}). It would get + no GRE neighbour entry and no mark rule, so its traffic would never enter + the tunnel - silently. Widen vpn_client_cidr, or drop this host from the + vpn_clients group. + success_msg: "{{ inventory_hostname }} {{ vpn_pf_inside.stdout | trim }}" + when: + - vpn_pf_is_client | bool + - vpn_pf_cidrs | length > 0 + tags: [preflight, always] + +# --------------------------------------------------------------------------- +# 4. Reachability: UDP 500 and 4500, both directions +# --------------------------------------------------------------------------- +# Only possible when the peer is a gateway in THIS inventory AND in THIS run. +# A managed cloud gateway has nothing to run a listener on, and a peer left +# out by --limit cannot start one, so both cases skip with an explanation. +# For a cloud peer the proof is the SA establishing - the role asserts that. + +- name: "Preflight: find the peer among the gateways in this run" + vars: + _addrs: >- + {{ (vpn_remote_addrs | default([])) + + ([vpn_remote_gw_ip] if vpn_remote_gw_ip is defined else []) }} + _in_inventory: >- + {{ groups['vpn_gateways'] | default([]) + | select('ne', inventory_hostname) + | map('extract', hostvars) + | selectattr('ansible_host', 'defined') + | selectattr('ansible_host', 'in', _addrs) + | map(attribute='inventory_hostname') | list }} + ansible.builtin.set_fact: + vpn_pf_peer_in_inventory: "{{ _in_inventory | first | default('') }}" + vpn_pf_peer: >- + {{ _in_inventory | select('in', ansible_play_hosts_all | default([])) + | list | first | default('') }} + when: vpn_pf_is_gateway | bool + tags: [preflight, always] + +- name: "Preflight: is a tunnel already established?" + ansible.builtin.shell: + cmd: | + command -v swanctl >/dev/null 2>&1 || { echo 0; exit 0; } + swanctl --list-sas 2>/dev/null | grep -c INSTALLED || echo 0 + register: vpn_pf_sas + changed_when: false + check_mode: false + when: vpn_pf_is_gateway | bool + tags: [preflight, always] + +- name: "Preflight: install the UDP probe" + ansible.builtin.template: + src: preflight-probe.sh.j2 + dest: /usr/local/sbin/vpn-preflight-probe.sh + mode: "0755" + when: + - vpn_pf_is_gateway | bool + - vpn_preflight_udp_test | bool + tags: [preflight, always] + +- name: "Preflight: decide whether the UDP probe is needed" + ansible.builtin.set_fact: + # Skip when the peer is not ours to instrument, when it is not in this + # run, or when a tunnel is already up - an established SA is stronger + # evidence than any probe. + vpn_pf_probe: >- + {{ (vpn_preflight_udp_test | bool) + and (vpn_pf_peer | default('') | length) > 0 + and ((vpn_pf_sas.stdout | default('0') | trim | int) == 0) }} + when: vpn_pf_is_gateway | bool + tags: [preflight, always] + +- name: "Preflight: explain what the reachability check will do" + ansible.builtin.debug: + msg: >- + {% if vpn_pf_probe | bool %} + {{ inventory_hostname }}: probing UDP 500 and 4500 to and from + {{ vpn_pf_peer }}. + {% elif (vpn_pf_sas.stdout | default('0') | trim | int) > 0 %} + {{ inventory_hostname }}: {{ vpn_pf_sas.stdout | trim }} SA(s) already + established, so UDP 500/4500 demonstrably works. Skipping the probe. + {% elif (vpn_pf_peer_in_inventory | default('') | length) > 0 %} + {{ inventory_hostname }}: the peer {{ vpn_pf_peer_in_inventory }} is in + the inventory but not in this run (--limit), so no listener can be + started on it. Skipping the probe; run without --limit to test both + directions. + {% elif not (vpn_preflight_udp_test | bool) %} + {{ inventory_hostname }}: UDP probe disabled by vpn_preflight_udp_test. + {% else %} + {{ inventory_hostname }}: the peer is not a host in this inventory (a + managed cloud gateway), so there is nothing to run a listener on. UDP + 500/4500 in both directions is still REQUIRED - the role asserts the SA + comes up, and an IKE timeout there almost always means one direction is + blocked. + {% endif %} + when: vpn_pf_is_gateway | bool + tags: [preflight, always] + +# These three tasks must stay in this order and in this play, and the play +# pins strategy: linear. Ansible then finishes each task on every host before +# starting the next, which is exactly the barrier the probe needs - every +# listener is up before any probe is sent. +- name: "Preflight: start UDP listeners on every gateway" + ansible.builtin.command: + cmd: /usr/local/sbin/vpn-preflight-probe.sh listen + register: vpn_pf_listen + changed_when: false + when: vpn_pf_probe | default(false) | bool + tags: [preflight, always] + +- name: "Preflight: send probes to the peer" + ansible.builtin.command: + cmd: >- + /usr/local/sbin/vpn-preflight-probe.sh send + {{ hostvars[vpn_pf_peer].ansible_host }} + vpnpf-{{ inventory_hostname }} + register: vpn_pf_send + changed_when: false + when: vpn_pf_probe | default(false) | bool + tags: [preflight, always] + +- name: "Preflight: check what arrived" + ansible.builtin.command: + cmd: >- + /usr/local/sbin/vpn-preflight-probe.sh check + vpnpf-{{ hostvars[vpn_pf_peer].inventory_hostname }} + register: vpn_pf_check + changed_when: false + when: vpn_pf_probe | default(false) | bool + tags: [preflight, always] + +- name: "Preflight: refuse to deploy through a one-way UDP block" + ansible.builtin.assert: + that: + - "'BLOCKED' not in vpn_pf_check.stdout" + - "'no-result' not in vpn_pf_check.stdout" + fail_msg: >- + UDP from {{ vpn_pf_peer }} to {{ inventory_hostname }} is BLOCKED. + Per-port result: {{ vpn_pf_check.stdout_lines | join('; ') }}. + IPsec needs UDP 500 AND 4500 open in BOTH directions between every + gateway public IP. Open them in the Crusoe VPC firewall (and on the peer + side) and re-run. Deploying now would present as an IKE timeout with + nothing useful in the strongSwan log. + success_msg: >- + UDP {{ vpn_pf_peer }} -> {{ inventory_hostname }}: + {{ vpn_pf_check.stdout_lines | join('; ') }} + when: vpn_pf_probe | default(false) | bool + tags: [preflight, always] + +- name: "Preflight: stop the listeners" + ansible.builtin.shell: + cmd: | + # Remove the state FIRST, so nothing below can prevent it. + rm -rf /run/vpn-preflight 2>/dev/null || true + # Then the listeners. Their command line is "python3 - + # /run/vpn-preflight/ ", so match the state path with a + # digit after the slash. The "[0-9]" is load-bearing: pkill -f tests + # every process's command line INCLUDING this shell's, and this text + # contains "/run/vpn-preflight" followed by a space or a quote, never a + # digit - so the pattern cannot match itself. A pattern that appears + # literally in its own command (or in a comment next to it) kills the + # shell running it, the task dies with rc -15, and failed_when hides it. + pkill -f '/run/vpn-preflight/[0-9]' 2>/dev/null || true + exit 0 + changed_when: false + # Cleanup must never fail a deployment. The listeners also carry their own + # timeout, so they exit on their own even if the kill does nothing. + failed_when: false + when: + - vpn_pf_is_gateway | bool + - vpn_preflight_udp_test | bool + tags: [preflight, always] + +- name: "Preflight: passed" + ansible.builtin.debug: + msg: >- + Preflight passed on {{ inventory_hostname }}. Continuing to the kernel + and configuration plays. + tags: [preflight, always] diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.service.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.service.j2 new file mode 100644 index 0000000..b4205bc --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.service.j2 @@ -0,0 +1,8 @@ +# Managed by Ansible - {{ inventory_hostname }} +[Unit] +Description=VPN ECMP health (prune dead tunnels, signal health to clients) +After=strongswan.service vpn-network.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/vpn-ecmp-health.sh diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.sh.j2 new file mode 100644 index 0000000..4b53e36 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.sh.j2 @@ -0,0 +1,75 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# Two jobs, both needed because an XFRM interface stays UP whether or not its +# SA is established: +# +# 1. Rebuild the local ECMP route from the tunnels the kernel actually has +# an SA for, so a dead tunnel stops attracting its share of traffic. +# Secondary when BGP is enabled, since BGP withdraws routes itself. +# +# 2. Signal this gateway's health to CLIENTS. Clients do not run BGP, so an +# ICMP probe to this gateway's own address is how they learn. Echo-request +# is dropped while no tunnel is up, which turns "VM alive but tunnel dead" +# into something a client can see and route around. +set -u + +LOCAL_IP="{{ vpn_local_gw_ip }}" +STATE_FILE=/run/vpn-ecmp-health.state + +alive=() +alive_count=0 +xfrm_state=$(ip -o xfrm state 2>/dev/null || true) + +{% for t in vpn_tunnels %} +# tunnel {{ t.index }} -> {{ t.xfrm_iface }} (if_id {{ t.if_id }}) +if printf '%s' "$xfrm_state" | grep -qE 'if_id ({{ "0x%x" | format(t.if_id) }}|{{ t.if_id }})([^0-9a-fA-F]|$)'; then + alive+=("{{ t.xfrm_iface }}") + alive_count=$(( alive_count + 1 )) +fi +{% endfor %} + +# --------------------------------------------------------------------------- +# 1. Rebuild the ECMP route when the live set of tunnels changes +# --------------------------------------------------------------------------- +{% if not (vpn_bgp_enabled | bool) and vpn_remote_subnet != "0.0.0.0/0" %} +REMOTE_SUBNET="{{ vpn_remote_subnet }}" + +if [ "$alive_count" -eq 0 ]; then + echo "ecmp-health: no established SAs - leaving the route alone rather than blackholing it" >&2 +else + desired="${alive[*]}" + previous=$(cat "$STATE_FILE" 2>/dev/null || true) + if [ "$desired" != "$previous" ]; then + args=() + for dev in "${alive[@]}"; do + args+=(nexthop dev "$dev" weight 1) + done + if ip route replace "$REMOTE_SUBNET" "${args[@]}"; then + printf '%s' "$desired" > "$STATE_FILE" + echo "ecmp-health: $REMOTE_SUBNET now via $desired" + fi + fi +fi +{% else %} +# BGP owns the data prefixes here, or this is a full-tunnel deployment where +# the mark table carries the default. Nothing to rebuild. +: +{% endif %} + +# --------------------------------------------------------------------------- +# 2. Client-visible health +# --------------------------------------------------------------------------- +RULE=(INPUT -p icmp --icmp-type echo-request -d "$LOCAL_IP" -j DROP) + +if [ "$alive_count" -gt 0 ]; then + # Healthy: make sure we answer probes. + while iptables -C "${RULE[@]}" 2>/dev/null; do + iptables -D "${RULE[@]}" || break + done +else + # Unhealthy: go quiet so clients drop us from their nexthop list. + iptables -C "${RULE[@]}" 2>/dev/null || iptables -I "${RULE[@]}" +fi + +exit 0 diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.timer.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.timer.j2 new file mode 100644 index 0000000..dc36d1c --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/ecmp-health.timer.j2 @@ -0,0 +1,11 @@ +# Managed by Ansible - {{ inventory_hostname }} +[Unit] +Description=Run VPN ECMP health every {{ vpn_ecmp_healthcheck_interval }} + +[Timer] +OnBootSec=30s +OnUnitActiveSec={{ vpn_ecmp_healthcheck_interval }} +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr-daemons.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr-daemons.j2 new file mode 100644 index 0000000..62e6c84 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr-daemons.j2 @@ -0,0 +1,23 @@ +# Managed by Ansible - {{ inventory_hostname }} +# Only zebra and bgpd. Everything else stays off. +zebra=yes +bgpd=yes +ospfd=no +ospf6d=no +ripd=no +ripngd=no +isisd=no +pimd=no +ldpd=no +nhrpd=no +eigrpd=no +babeld=no +sharpd=no +pbrd=no +bfdd=no +fabricd=no +vrrpd=no +pathd=no + +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr.conf.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr.conf.j2 new file mode 100644 index 0000000..ca129cc --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/frr.conf.j2 @@ -0,0 +1,58 @@ +! Managed by Ansible - {{ inventory_hostname }} +! +! One BGP session per IPsec tunnel. Azure REQUIRES BGP whenever several +! on-premises devices advertise the same prefixes to one VPN gateway - static +! routing cannot express it. Every gateway advertises the SAME prefixes, which +! is what makes the peer ECMP back across all of them. +! +frr defaults traditional +hostname {{ inventory_hostname }} +log syslog informational +service integrated-vtysh-config +! +router bgp {{ vpn_bgp_local_asn }} + bgp router-id {{ vpn_local_gw_ip }} + no bgp default ipv4-unicast + neighbor PEER peer-group + neighbor PEER remote-as {{ vpn_bgp_peer_asn }} + ! The peer is reached across the tunnel via a host route, not a connected + ! subnet, so multihop is required or the session never establishes. + neighbor PEER ebgp-multihop {{ vpn_bgp_ebgp_multihop }} + ! Must match the BgpPeerIpAddress configured on this gateway's Azure Local + ! Network Gateway, exactly. + neighbor PEER update-source {{ vpn_local_gw_ip }} + neighbor PEER timers {{ vpn_bgp_keepalive }} {{ vpn_bgp_holdtime }} +{% for t in vpn_tunnels if t.bgp_peer %} + neighbor {{ t.bgp_peer }} peer-group PEER +{% endfor %} + ! + address-family ipv4 unicast +{% for p in vpn_bgp_advertise %} + network {{ p }} +{% endfor %} + maximum-paths {{ vpn_bgp_maximum_paths }} + neighbor PEER activate + neighbor PEER soft-reconfiguration inbound + neighbor PEER route-map PEER-IN in + neighbor PEER route-map PEER-OUT out + exit-address-family +! +{% for p in vpn_bgp_advertise %} +ip prefix-list ADVERTISE seq {{ loop.index * 5 }} permit {{ p }} +{% endfor %} +{% for p in vpn_bgp_accept %} +ip prefix-list ACCEPT seq {{ loop.index * 5 }} permit {{ p }} le 32 +{% endfor %} +! +! Filters in BOTH directions are not optional hygiene. Without an outbound +! filter a gateway can leak a default route to the peer; without an inbound +! one the peer can install a route that swallows this gateway's own +! management path. +route-map PEER-OUT permit 10 + match ip address prefix-list ADVERTISE +! +route-map PEER-IN permit 10 + match ip address prefix-list ACCEPT +! +line vty +! diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/perf-tuning.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/perf-tuning.sh.j2 new file mode 100644 index 0000000..e9fdefe --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/perf-tuning.sh.j2 @@ -0,0 +1,171 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# Host tuning that cannot live in sysctl.d: NIC queues, offloads, RPS/RFS, +# IRQ affinity, CPU governor. Idempotent; safe to re-run. +# +# None of this raises the Azure-side ceiling. It stops the Crusoe side from +# falling short of it. Every value here was measured; see the role defaults +# for the numbers and AZURE-10G-GUIDE.md for the deployment recipe. +set -u + +UPLINK="{{ vpn_primary_iface }}" +NCPU=$(nproc) +{% if vpn_rx_queues %} +WANT_QUEUES={{ vpn_rx_queues }} +{% else %} +WANT_QUEUES="$NCPU" +{% endif %} + +log() { echo "perf-tuning: $*"; } + +# ---------- combined queues ---------- +# Queue count is not the independent variable - FLOW count is. RSS hashes +# flows into queues, so extra queues only help when there are extra flows. +# Raised anyway, because it costs nothing and the clients do have many flows. +MAXQ=$(ethtool -l "$UPLINK" 2>/dev/null | awk '/^Combined:/{print $2; exit}') +if [ -n "${MAXQ:-}" ] && [ "$MAXQ" -gt 0 ] 2>/dev/null; then + [ "$WANT_QUEUES" -gt "$MAXQ" ] && WANT_QUEUES="$MAXQ" + ethtool -L "$UPLINK" combined "$WANT_QUEUES" 2>/dev/null \ + && log "combined queues -> $WANT_QUEUES (NIC max $MAXQ)" \ + || log "could not set combined queues (often fixed by the hypervisor)" +else + log "NIC does not report combined queues - skipping" +fi + +{% if vpn_nic_rings | int > 0 %} +# ---------- ring sizes ---------- +# Explicitly requested. Measured NEGATIVE on Crusoe mlx5 (rx/tx 8192 gave +# 4.24 Gbps against a 4.67 Gbps default-ring baseline, -9.1%): a deeper ring +# widens reordering, which feeds ESP anti-replay drops. Measure before +# keeping this. +ethtool -G "$UPLINK" rx {{ vpn_nic_rings }} tx {{ vpn_nic_rings }} 2>/dev/null \ + && log "rings -> {{ vpn_nic_rings }} rx/tx (measured negative here - verify)" \ + || log "could not set ring sizes" +{% endif %} + +# ---------- offloads ---------- +# ESP GSO/GRO lets the stack hand large segments to the crypto layer instead +# of paying per-packet cost. Saves CPU; does not change the wire packet rate. +ethtool -K "$UPLINK" {{ vpn_nic_offloads }} 2>/dev/null \ + && log "offloads: {{ vpn_nic_offloads }}" || log "some offloads unsupported" +ethtool -k "$UPLINK" 2>/dev/null | grep -i esp | sed 's/^/perf-tuning: esp-offload: /' || true +{% if vpn_adaptive_rx | bool and vpn_rx_usecs | int == 0 %} +# Skipped when vpn_rx_usecs is set: that asks for FIXED coalescing instead. +ethtool -C "$UPLINK" adaptive-rx on 2>/dev/null && log "adaptive-rx on" || true +{% endif %} + +{% if vpn_enable_rps | bool %} +# ---------- RPS / RFS ---------- +# Re-queues packets to other CPUs early in netif_receive_skb, BEFORE the XFRM +# input path - so ESP decrypt spreads across cores even on a single RX queue. +# This is the fallback that works regardless of how the hypervisor hashes. +MASK=$(python3 -c "print(format((1 << $NCPU) - 1, 'x'))") +for q in /sys/class/net/"$UPLINK"/queues/rx-*; do + [ -e "$q/rps_cpus" ] && echo "$MASK" > "$q/rps_cpus" 2>/dev/null || true +{% if not vpn_disable_rfs | bool %} + # Only when RFS is explicitly wanted; it reorders a single SA's packets. + [ -e "$q/rps_flow_cnt" ] && echo 4096 > "$q/rps_flow_cnt" 2>/dev/null || true +{% endif %} +done +log "RPS enabled across $NCPU CPUs" +{% endif %} + +{% if vpn_disable_rfs | bool %} +# ---------- disable RFS ---------- +# RFS steers a packet to the CPU where the consuming socket last ran. For a +# single IPsec tunnel that hands the SA's packets between CPUs, so they reach +# xfrm_input out of order and the anti-replay window rejects the stragglers. +# The tunnel stays up and ping stays clean while TCP retransmits. +# +# Measured, one tunnel, 16 flows, 54 ms RTT: +# rps_flow_cnt 4096 -> 4.67 Gbps, 18,624 retransmits +# rps_flow_cnt 0 -> 4.86 Gbps, 106 retransmits +n=0 +for q in /sys/class/net/"$UPLINK"/queues/rx-*; do + [ -e "$q/rps_flow_cnt" ] || continue + echo 0 > "$q/rps_flow_cnt" 2>/dev/null && n=$(( n + 1 )) || true +done +log "RFS disabled on $n RX queues (rps_flow_cnt=0)" +# Global flow table too, so nothing re-populates it. +sysctl -qw net.core.rps_sock_flow_entries=0 2>/dev/null || true +{% endif %} + +{% if vpn_rx_usecs | int > 0 %} +# ---------- fixed interrupt coalescing ---------- +# Batches more packets per NAPI poll. Measured +2.6% at 64 us; 128 us +# measured -2.0%, so 64 is the useful setting and higher is not. +ethtool -C "$UPLINK" adaptive-rx off rx-usecs {{ vpn_rx_usecs }} rx-frames 128 \ + 2>/dev/null && log "coalescing -> fixed, rx-usecs {{ vpn_rx_usecs }}" \ + || log "could not set fixed coalescing" +{% endif %} + +{% if vpn_gro_normal_batch | int > 0 %} +# ---------- GRO batch ---------- +sysctl -qw net.core.gro_normal_batch={{ vpn_gro_normal_batch }} 2>/dev/null \ + && log "gro_normal_batch -> {{ vpn_gro_normal_batch }}" || true +{% endif %} + +{% if vpn_irq_affinity | bool %} +# ---------- IRQ affinity + XPS ---------- +# Keeps a flow's receive and transmit work on one core. irqbalance moving NIC +# IRQs around costs cache locality, so it is stopped rather than fought. +systemctl stop irqbalance 2>/dev/null || true +systemctl disable irqbalance 2>/dev/null || true +i=0 +for irq in $(grep -i "$UPLINK" /proc/interrupts | awk -F: '{print $1}' | tr -d ' '); do + cpu=$(( i % NCPU )) + echo "$cpu" > "/proc/irq/$irq/smp_affinity_list" 2>/dev/null || true + i=$(( i + 1 )) +done +log "pinned $i NIC IRQs across $NCPU CPUs" +i=0 +for q in /sys/class/net/"$UPLINK"/queues/tx-*; do + [ -e "$q/xps_cpus" ] || continue + printf '%x\n' $(( 1 << (i % NCPU) )) > "$q/xps_cpus" 2>/dev/null || true + i=$(( i + 1 )) +done +log "set xps_cpus on $i TX queues" +{% endif %} + +{% if vpn_cpu_governor %} +# ---------- CPU governor ---------- +# Crypto-heavy softirq work is exactly what frequency scaling handles badly. +if command -v cpupower >/dev/null 2>&1; then + cpupower frequency-set -g {{ vpn_cpu_governor }} >/dev/null 2>&1 \ + && log "governor -> {{ vpn_cpu_governor }}" || true +else + for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + [ -e "$g" ] && echo {{ vpn_cpu_governor }} > "$g" 2>/dev/null || true + done +fi +{% endif %} + +# ---------- report: which AES-GCM implementation is actually bound ---------- +# The kernel binds the highest-priority driver registered for +# rfc4106(gcm(aes)). Before 6.11 that is only the 2010-era AES-NI code. +# 6.11+ adds a VAES/AVX-512 one worth ~+23% per tunnel on Zen 4/5. +grep -qw aes /proc/cpuinfo \ + && log "CPU reports AES-NI" \ + || log "WARNING: no AES-NI - GCM will be an order of magnitude slower" + +# modprobe so the drivers are registered before we look. +modprobe aesni_intel 2>/dev/null || true +# /proc/crypto also lists internal instances ("__rfc4106(...)") and wrappers +# ("seqiv(rfc4106(...))"). Anchoring the name picks the entry consumers bind. +DRV=$(sed -n "/^name .*: rfc4106(gcm(aes))\$/,+1p" /proc/crypto 2>/dev/null \ + | sed -n "s/^driver *: *//p" | head -1) +if [ -n "${DRV:-}" ]; then + log "kernel AES-GCM driver: $DRV" + case "$DRV" in + *vaes*) log " VAES/AVX-512 in use - this is the fast path" ;; + *) log " SLOW PATH: no VAES driver. Kernel $(uname -r) predates it." + log " Measured +23% per tunnel on 6.11 vs 6.8. See AZURE-10G-GUIDE.md." ;; + esac + if grep -qw vaes /proc/cpuinfo && [ "${DRV#*vaes}" = "$DRV" ]; then + log " NOTE: this CPU HAS vaes but the kernel is not using it." + fi +else + log "note: rfc4106(gcm(aes)) not registered yet (bound on first SA)" +fi +exit 0 diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/preflight-probe.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/preflight-probe.sh.j2 new file mode 100644 index 0000000..7142e1e --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/preflight-probe.sh.j2 @@ -0,0 +1,147 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# UDP reachability probe for the IKE ports, driven by the preflight play. +# +# Why this exists: IPsec needs UDP 500 and 4500 to pass in BOTH directions, +# and a one-way block presents as "IKE timeout" with nothing useful in the +# strongSwan log. This turns that into a named failure before anything is +# configured. +# +# Modes: +# listen bind each port (only if free) and record the first datagram +# send send a tagged datagram to on each port +# check report, per port, whether that tag arrived +# ports report which ports are already in use +# +# The listener is a background python3 process with a timeout, so it cannot +# outlive the play even if a probe never arrives. +# +# State lives under /run, root-only (0700), and result files are opened with +# O_NOFOLLOW|O_CREAT|O_TRUNC at 0600. /tmp would let any local user pre-plant +# a symlink and turn "root writes a result file" into "root writes wherever +# I point it". +set -u + +PORTS="500 4500" +STATE=/run/vpn-preflight +LISTEN_SECS={{ vpn_preflight_listen_secs }} + +state_dir() { + mkdir -m 0700 -p "$STATE" 2>/dev/null || true + if [ "$(stat -c %u "$STATE" 2>/dev/null)" != "0" ]; then + echo "refusing to use $STATE: not owned by root" >&2 + exit 3 + fi + chmod 0700 "$STATE" 2>/dev/null || true +} + +port_busy() { + # "ss -lun" prints State Recv-Q Send-Q Local:Port Peer:Port [Process] - + # no Netid column when a single protocol is selected - so the local + # socket is column 4. -H drops the header. + ss -lunH 2>/dev/null | awk '{print $4}' | grep -qE "[:.]$1\$" +} + +case "${1:-}" in + +ports) + for p in $PORTS; do + if port_busy "$p"; then echo "$p busy"; else echo "$p free"; fi + done + ;; + +listen) + state_dir + for p in $PORTS; do + rm -f "$STATE/$p" + if port_busy "$p"; then + # Almost always charon from a previous run. Not a problem: if the + # tunnel is already up, UDP has already proven it works, and the + # play skips the probe on that basis. + echo "BUSY:$p" > "$STATE/$p" + continue + fi + nohup python3 - "$p" "$STATE/$p" "$LISTEN_SECS" >/dev/null 2>&1 <<'PYEOF' & +import os, socket, sys +port, out, secs = int(sys.argv[1]), sys.argv[2], float(sys.argv[3]) + +def write(text): + fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + try: + os.write(fd, text.encode("utf-8", "replace")) + finally: + os.close(fd) + +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +try: + s.bind(("0.0.0.0", port)) +except OSError as e: + write("BINDFAIL:%s" % e) + raise SystemExit(0) +s.settimeout(secs) +try: + data, addr = s.recvfrom(256) + write("GOT:%s:%s" % (data.decode("utf-8", "replace"), addr[0])) +except Exception: + write("TIMEOUT") +PYEOF + done + sleep 1 + for p in $PORTS; do + if [ -s "$STATE/$p" ] && grep -q '^BUSY' "$STATE/$p"; then + echo "$p busy-skipped" + elif port_busy "$p"; then + echo "$p listening" + else + echo "$p bind-failed" + fi + done + ;; + +send) + DEST="${2:?send needs a destination IP}" + TAG="${3:?send needs a tag}" + for p in $PORTS; do + python3 - "$DEST" "$p" "$TAG" <<'PYEOF' +import socket, sys +dest, port, tag = sys.argv[1], int(sys.argv[2]), sys.argv[3] +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.settimeout(3) +try: + # Sent three times: a single UDP datagram has no retransmit of its own, + # and one drop would read as a firewall block. + for _ in range(3): + s.sendto(tag.encode(), (dest, port)) + print("%s sent" % port) +except Exception as e: + print("%s send-failed %s" % (port, e)) +PYEOF + done + ;; + +check) + TAG="${2:?check needs a tag}" + for p in $PORTS; do + f="$STATE/$p" + if [ ! -f "$f" ]; then + echo "$p no-result" + elif grep -q '^BUSY' "$f"; then + echo "$p busy-skipped" + elif grep -q "GOT:$TAG" "$f"; then + echo "$p PASS" + elif grep -q '^BINDFAIL' "$f"; then + echo "$p bind-failed" + else + echo "$p BLOCKED" + fi + done + ;; + +*) + echo "usage: $0 {ports|listen|send |check }" >&2 + exit 2 + ;; +esac +exit 0 diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/strongswan-vpn.conf.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/strongswan-vpn.conf.j2 new file mode 100644 index 0000000..0391d31 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/strongswan-vpn.conf.j2 @@ -0,0 +1,10 @@ +# Managed by Ansible - vpn_gateway role +# +# This role owns the routing table by hand (see vpn-network.sh), and when BGP +# is enabled FRR owns the learned prefixes. Letting charon install its own +# per-SA routes would fight the ECMP entry. +charon { + install_routes = no + install_virtual_ip = no + threads = {{ vpn_charon_threads | default(32) }} +} diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/swanctl-vpn.conf.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/swanctl-vpn.conf.j2 index 8c6bf30..2c965ad 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/swanctl-vpn.conf.j2 +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/swanctl-vpn.conf.j2 @@ -1,32 +1,58 @@ -# Managed by Ansible — {{ inventory_hostname }} +# Managed by Ansible - {{ inventory_hostname }} +{% if vpn_tunnels | length > 1 %} +# +# {{ vpn_tunnels | length }} parallel IPsec SAs to the same remote site. +# All children negotiate IDENTICAL traffic selectors on purpose - the if_id is +# what keeps the kernel policies distinct, and a cloud peer only ECMPs back +# when every connection advertises the same address space. +{% endif %} connections { - site-vpn { +{% for t in vpn_tunnels %} + {{ t.name }} { version = 2 - local_addrs = %any - remote_addrs = {{ vpn_remote_gw_ip }} + local_addrs = {{ t.local_addr }} + remote_addrs = {{ t.remote_addr }} +{% if vpn_force_encap | bool %} + encap = yes +{% endif %} +{% if vpn_tunnels | length > 1 %} + # Several SAs legitimately share these identities; without this charon + # may treat them as duplicates and tear down all but one. + unique = never +{% else %} + # One tunnel per identity pair, so a reconnect should REPLACE the old + # SA rather than sit alongside it. NOTE, measured: this helps but does + # not fully prevent duplicates, because "unique" is a RESPONDER-side + # rule and a redeploy can initiate the duplicate locally. Cosmetic - + # traffic uses one set of kernel states either way. + unique = replace +{% endif %} local { auth = psk - id = {{ vpn_local_id }} + id = {{ t.local_id }} } remote { auth = psk - id = {{ vpn_remote_id }} + id = {{ t.remote_id }} } children { - site-tunnel { - local_ts = {{ vpn_local_subnet }} - remote_ts = {{ vpn_remote_subnet }} - start_action = trap + {{ t.child }} { + local_ts = {{ vpn_child_local_ts }} + remote_ts = {{ vpn_child_remote_ts }} + start_action = {{ vpn_start_action_effective }} close_action = start dpd_action = restart esp_proposals = {{ vpn_esp_proposals }} mode = tunnel - reqid = {{ vpn_xfrm_if_id }} - if_id_in = {{ vpn_xfrm_if_id }} - if_id_out = {{ vpn_xfrm_if_id }} + reqid = {{ t.if_id }} + if_id_in = {{ t.if_id }} + if_id_out = {{ t.if_id }} rekey_time = {{ vpn_child_rekey_time }} + # Multi-queue receive reorders packets; the default window + # drops reordered-but-valid packets at high pps. + replay_window = {{ vpn_replay_window }} } } @@ -34,12 +60,17 @@ connections { dpd_delay = {{ vpn_dpd_delay }} rekey_time = {{ vpn_ike_rekey_time }} } +{% endfor %} } secrets { ike-site { - id-a = {{ vpn_local_id }} - id-b = {{ vpn_remote_id }} +{% for id in vpn_tunnels | map(attribute='local_id') | unique %} + id-local-{{ loop.index }} = {{ id }} +{% endfor %} +{% for id in vpn_tunnels | map(attribute='remote_id') | unique %} + id-remote-{{ loop.index }} = {{ id }} +{% endfor %} secret = "{{ vpn_psk }}" } } diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/sysctl-vpn.conf.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/sysctl-vpn.conf.j2 index 905f4da..7c00e43 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/sysctl-vpn.conf.j2 +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/sysctl-vpn.conf.j2 @@ -1,6 +1,52 @@ -# Managed by Ansible — vpn_gateway role +# Managed by Ansible - vpn_gateway role net.ipv4.ip_forward = 1 net.ipv4.conf.all.send_redirects = 0 net.ipv4.conf.default.send_redirects = 0 net.ipv4.conf.all.accept_redirects = 0 net.ipv4.conf.default.accept_redirects = 0 + +# --- ECMP --- +# 0 (kernel default) hashes source/destination IP only, so every flow between +# the same pair of hosts takes ONE nexthop and the other tunnels sit idle. +# The setup looks correct and performs like a single tunnel. +net.ipv4.fib_multipath_hash_policy = {{ vpn_multipath_hash_policy }} + +# With ECMP + XFRM the reverse path will not always match the forward nexthop. +net.ipv4.conf.all.rp_filter = {{ vpn_rp_filter }} +net.ipv4.conf.default.rp_filter = {{ vpn_rp_filter }} +{% if vpn_transport == 'gre_fou' %} + +# --- neighbour table --- +# The multipoint GRE tunnel holds one PERMANENT neighbour entry per host in +# vpn_client_cidr. The default hard limit (gc_thresh3) is 1024, so a /22 or +# larger silently overflows with "neighbour table overflow" and some clients +# simply never work. These values cover up to a /19. +net.ipv4.neigh.default.gc_thresh1 = {{ vpn_neigh_gc_thresh1 }} +net.ipv4.neigh.default.gc_thresh2 = {{ vpn_neigh_gc_thresh2 }} +net.ipv4.neigh.default.gc_thresh3 = {{ vpn_neigh_gc_thresh3 }} +{% endif %} +{% if vpn_perf_tuning | bool %} + +# --- Throughput tuning (vpn_perf_tuning) --- +net.core.rmem_max = {{ vpn_rmem_max }} +net.core.wmem_max = {{ vpn_wmem_max }} +net.core.netdev_max_backlog = 250000 +net.core.somaxconn = 4096 + +# Let softirq process more packets per poll before yielding. Helps at high pps. +net.core.netdev_budget = {{ vpn_netdev_budget }} +net.core.netdev_budget_usecs = {{ vpn_netdev_budget_usecs }} +{% endif %} +{% if vpn_enable_rps | bool and not vpn_disable_rfs | bool %} + +# Global RFS flow table. ONLY sized when RFS is explicitly wanted: steering a +# single IPsec SA's packets by flow hands them between CPUs, they reach +# xfrm_input out of order, and the anti-replay window drops the stragglers. +# Measured 18,624 retransmits with RFS on versus 106 with it off. +net.core.rps_sock_flow_entries = 32768 +{% endif %} +{% if vpn_disable_rfs | bool %} + +# RFS off. See vpn_disable_rfs in the role defaults for the measurement. +net.core.rps_sock_flow_entries = 0 +{% endif %} diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/tunnels.json.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/tunnels.json.j2 new file mode 100644 index 0000000..385d9ed --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/tunnels.json.j2 @@ -0,0 +1,105 @@ +{#- + Derive the tunnel list for this gateway. Rendered by tasks/main.yml via + lookup('template') and parsed with from_json, so all the fiddly derivation + lives in one readable place instead of a chain of set_fact expressions. + + Backward compatibility is load bearing here: with vpn_tunnel_count == 1 the + output must reproduce today's names exactly - connection "site-vpn", child + "site-tunnel", interface vpn_xfrm_iface, if_id vpn_xfrm_if_id, local_addrs + "%any", and the identities straight from vpn_local_id / vpn_remote_id. + + See README.md, "Multiple tunnels and multiple gateways". +-#} +{%- if vpn_tunnels_override | length > 0 -%} +{{ vpn_tunnels_override | to_json }} +{%- else -%} +{#- vpn_tunnel_count_effective resolves 0 (auto) to the number of remote + outer addresses; see tasks/main.yml. Clamped to at least 1 HERE as well, + not just in the task: a count of 0 renders an empty up_xfrm() bash + function, which is a syntax error, and this template is the single source + of truth for anything that reads the tunnel list. -#} +{%- set count = [ (vpn_tunnel_count_effective | default(vpn_tunnel_count) | int), 1 ] | max -%} + +{#- normalise vpn_local_addrs into dicts -#} +{%- set locals_ = [] -%} +{%- for a in vpn_local_addrs -%} + {%- if a is mapping -%} + {%- set _ = locals_.append({ + 'addr': a.addr, + 'public': a.public | default(a.addr), + 'iface': a.iface | default(vpn_primary_iface), + 'gateway': a.gateway | default('')}) -%} + {%- else -%} + {%- set _ = locals_.append({ + 'addr': a, 'public': a, + 'iface': vpn_primary_iface, 'gateway': ''}) -%} + {%- endif -%} +{%- endfor -%} +{%- if locals_ | length == 0 -%} + {#- No explicit local addresses: bind anywhere. Correct on Crusoe, where a + VM has one IP and the public address is 1:1 NAT'd to it. -#} + {%- set locals_ = [{'addr': '%any', 'public': '', + 'iface': vpn_primary_iface, 'gateway': ''}] -%} +{%- endif -%} + +{%- set remotes = vpn_remote_addrs if (vpn_remote_addrs | length > 0) + else [vpn_remote_gw_ip] -%} + +{#- identity style -#} +{%- if vpn_id_style -%} + {%- set style = vpn_id_style -%} +{%- elif count == 1 -%} + {%- set style = 'fixed' -%} +{%- elif vpn_local_addrs | length > 0 or remotes | length > 1 -%} + {#- Distinct remote outer IPs, one per tunnel: identify by address. This is + the Azure active-active case, where the peer matches on IP and the + "indexed" style would mangle an IP into "20.0.0.1-1". -#} + {%- set style = 'address' -%} +{%- else -%} + {%- set style = 'indexed' -%} +{%- endif -%} + +{%- set tunnels = [] -%} +{%- for i in range(count) -%} + {%- set l = locals_[i % (locals_ | length)] -%} + {%- set r = remotes[i % (remotes | length)] -%} + + {%- if style == 'address' -%} + {%- set lid = l.public if l.public else vpn_local_id -%} + {%- set rid = r -%} + {%- elif style == 'indexed' -%} + {#- The LOCAL identity is NOT indexed: this gateway is one machine and must + present the same identity on every tunnel, or the peer's remote-id match + fails. Only the REMOTE identity is indexed, because each tunnel goes to + a different peer gateway (site-b-gw-1, -2, -3 ...). -#} + {%- set lid = vpn_local_id -%} + {#- Only a name-style identity ("site-b-gw@x") can be indexed. The default + identity is now an IP address, and "20.1.2.3-1" is not one, so an IP + passes through untouched. -#} + {%- set rid = (vpn_remote_id | regex_replace('^([^@]+)', '\\1-' ~ (i + 1))) + if '@' in (vpn_remote_id | string) else vpn_remote_id -%} + {%- else -%} + {%- set lid = vpn_local_id -%} + {%- set rid = vpn_remote_id -%} + {%- endif -%} + + {%- set _ = tunnels.append({ + 'index': i, + 'name': 'site-vpn' if count == 1 else 'site-vpn-' ~ (i + 1), + 'child': 'site-tunnel' if count == 1 else 'site-tunnel-' ~ (i + 1), + 'xfrm_iface': vpn_xfrm_iface if count == 1 + else vpn_xfrm_iface_prefix ~ i, + 'if_id': (vpn_xfrm_if_id | int) + i, + 'local_addr': l.addr, + 'local_public': l.public, + 'local_iface': l.iface, + 'local_gateway': l.gateway, + 'remote_addr': r, + 'local_id': lid, + 'remote_id': rid, + 'bgp_peer': vpn_bgp_peer_addrs[i] + if (vpn_bgp_peer_addrs | length) > i else '', + 'srcrt_table': (vpn_srcrt_table_base | int) + i }) -%} +{%- endfor -%} +{{ tunnels | to_json }} +{%- endif -%} diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.service.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.service.j2 index 1322b87..265f62c 100644 --- a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.service.j2 +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.service.j2 @@ -1,6 +1,6 @@ -# Managed by Ansible — {{ inventory_hostname }} +# Managed by Ansible - {{ inventory_hostname }} [Unit] -Description=VPN Gateway Network +Description=VPN Gateway Network (XFRM interfaces, routes, marks) After=network-online.target strongswan.service Wants=network-online.target @@ -9,46 +9,8 @@ Type=oneshot RemainAfterExit=yes Restart=on-failure RestartSec=5 -ExecStart=/bin/bash -c '\ - set -e; \ - ip link del {{ vpn_xfrm_iface }} 2>/dev/null || true; \ - ip link add {{ vpn_xfrm_iface }} type xfrm dev {{ vpn_primary_iface }} if_id {{ vpn_xfrm_if_id }}; \ - ip link set {{ vpn_xfrm_iface }} mtu {{ vpn_xfrm_mtu }}; \ - ip link set {{ vpn_xfrm_iface }} up; \ -{% if vpn_remote_subnet != "0.0.0.0/0" %} - ip route replace {{ vpn_remote_subnet }} dev {{ vpn_xfrm_iface }}; \ -{% endif %} -{% if vpn_use_gre | default(false) %} - modprobe fou; \ - ip link del gre-vpn 2>/dev/null || true; \ - ip tunnel del gre-vpn 2>/dev/null || true; \ - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true; \ - ip fou add port {{ vpn_fou_port }} ipproto 47; \ - ip link add name gre-vpn type gre local {{ vpn_local_gw_ip }} key {{ vpn_gre_key }} ttl 255 \ - encap fou encap-sport {{ vpn_fou_port }} encap-dport {{ vpn_fou_port }}; \ - ip link set gre-vpn mtu {{ vpn_gre_mtu }}; \ - ip link set gre-vpn up; \ - python3 -c "import ipaddress, subprocess; gw=\"{{ vpn_local_gw_ip }}\"; [subprocess.run([\"ip\",\"neigh\",\"replace\",str(ip),\"lladdr\",str(ip),\"dev\",\"gre-vpn\",\"nud\",\"permanent\"], check=True) for ip in ipaddress.ip_network(\"{{ vpn_client_cidr }}\", strict=False).hosts() if str(ip) != gw]"; \ - ip rule del iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }} 2>/dev/null || true; \ - ip rule add iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }}; \ - ip route replace {{ vpn_client_cidr }} dev gre-vpn table {{ vpn_rt_table }}; \ - iptables -t mangle -D PREROUTING -i gre-vpn -j MARK --set-mark 0x64 2>/dev/null || true; \ - iptables -t mangle -A PREROUTING -i gre-vpn -j MARK --set-mark 0x64; \ - ip rule del fwmark 0x64 lookup 200 2>/dev/null || true; \ - ip rule add fwmark 0x64 lookup 200; \ - ip route replace default dev {{ vpn_xfrm_iface }} table 200; \ -{% endif %} - echo "VPN network ready"' - -ExecStop=/bin/bash -c '\ -{% if vpn_use_gre | default(false) %} - ip rule del iif {{ vpn_xfrm_iface }} lookup {{ vpn_rt_table }} 2>/dev/null || true; \ - ip route flush table {{ vpn_rt_table }} 2>/dev/null || true; \ - ip link del gre-vpn 2>/dev/null || true; \ - ip tunnel del gre-vpn 2>/dev/null || true; \ - ip fou del port {{ vpn_fou_port }} 2>/dev/null || true; \ -{% endif %} - ip link del {{ vpn_xfrm_iface }} 2>/dev/null || true' +ExecStart=/usr/local/sbin/vpn-network.sh up +ExecStop=/usr/local/sbin/vpn-network.sh down [Install] WantedBy=multi-user.target diff --git a/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.sh.j2 b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.sh.j2 new file mode 100644 index 0000000..c163e35 --- /dev/null +++ b/strongswan-ipsec/ansible/roles/vpn_gateway/templates/vpn-network.sh.j2 @@ -0,0 +1,335 @@ +#!/bin/bash +# Managed by Ansible - {{ inventory_hostname }} +# +# Single source of truth for every link, route, rule and mark this gateway +# needs. Called by the Ansible play AND by vpn-network.service, so live state +# and boot state cannot drift. +# +# Usage: vpn-network.sh {up|down} +set -u + +UPLINK="{{ vpn_primary_iface }}" +LOCAL_IP="{{ vpn_local_gw_ip }}" +XFRM_MTU="{{ vpn_xfrm_mtu }}" + +{# ---------- Jinja-side derivations ---------- #} +{%- if vpn_tunnels | length == 1 -%} +{%- set XFRM_NH = 'dev ' ~ vpn_tunnels[0].xfrm_iface -%} +{%- else -%} +{%- set XFRM_NH = vpn_tunnels | map(attribute='xfrm_iface') + | map('regex_replace', '^(.*)$', 'nexthop dev \\1 weight 1') | join(' ') -%} +{%- endif -%} +{%- set gre_devs = [] -%} +{%- if vpn_transport == 'gre_fou' -%} +{%- for p in range(vpn_fou_port_count | int) -%} +{%- set _ = gre_devs.append({ + 'dev': 'gre-vpn' if (vpn_fou_port_count | int) == 1 else 'gre-vpn' ~ p, + 'port': (vpn_fou_port | int) + p, + 'key': (vpn_gre_key | int) + p}) -%} +{%- endfor -%} +{%- endif -%} +{%- if gre_devs | length == 1 -%} +{%- set GRE_NH = 'dev ' ~ gre_devs[0].dev -%} +{%- elif gre_devs | length > 1 -%} +{%- set GRE_NH = gre_devs | map(attribute='dev') + | map('regex_replace', '^(.*)$', 'nexthop dev \\1 weight 1') | join(' ') -%} +{%- else -%} +{%- set GRE_NH = '' -%} +{%- endif -%} +{%- set MSS_ARG = '--set-mss ' ~ (vpn_mss_effective | trim) %} + +# transport = {{ vpn_transport }}, tunnels = {{ vpn_tunnels | length }}, bgp = {{ vpn_bgp_enabled | bool }} +{% for t in vpn_tunnels %} +# tunnel {{ t.index }}: {{ t.xfrm_iface }} if_id {{ t.if_id }} on {{ t.local_iface }}, {{ t.local_addr }} -> {{ t.remote_addr }} +{% endfor %} + +# =========================================================================== +# up +# =========================================================================== + +up_xfrm() { + # Purge stale xfrm interfaces AND their routing rules, left behind when the + # tunnel count SHRINKS - reconfiguring a 5-tunnel mesh down to the 2 an + # Azure peer allows, for instance. Two reasons this matters: + # * an "iif" rule OUTLIVES its device, showing as "iif xfrmN [detached]" + # * fib_rules_lookup walks every rule for every packet, and that showed + # up as a real per-packet cost in a perf profile of the receive path + _keep=" {% for t in vpn_tunnels %}{{ t.xfrm_iface }} {% endfor %}" + # Rules first, deleted by PREFERENCE NUMBER: "ip rule del iif " fails + # once the device is gone, and iproute2 prints such rules as + # "iif xfrmN [detached] lookup 100" - the optional "[detached]" is matched + # here so exactly that case is caught. + ip rule show 2>/dev/null \ + | sed -n 's/^\([0-9]*\):[[:space:]]*from all iif \([^ ]*\)\( \[detached\]\)\{0,1\} lookup {{ vpn_rt_table }}.*/\1 \2/p' \ + | while read -r _prio _dev; do + case "$_keep" in *" $_dev "*) continue ;; esac + ip rule del pref "$_prio" 2>/dev/null || true + done + # Then every XFRM-type device not in the keep list, WHATEVER its name - + # keying on the name prefix would leave a device from an older + # configuration (different prefix) holding an if_id. A VPN gateway is a + # dedicated VM, so any other xfrm interface on it is stale by definition. + for _d in $(ip -o link show type xfrm 2>/dev/null | awk -F': ' '{print $2}' \ + | sed 's/@.*//'); do + case "$_keep" in *" $_d "*) continue ;; esac + ip link del "$_d" 2>/dev/null || true + done +{% for t in vpn_tunnels %} + ip link del {{ t.xfrm_iface }} 2>/dev/null || true + ip link add {{ t.xfrm_iface }} type xfrm dev {{ t.local_iface }} if_id {{ t.if_id }} + ip link set {{ t.xfrm_iface }} mtu "$XFRM_MTU" + ip link set {{ t.xfrm_iface }} up +{% endfor %} +} + +# Per-uplink source routing. Without this every tunnel's ESP would egress the +# primary NIC regardless of its source address, and extra vNICs carry nothing. +up_source_routing() { +{% set srcrt = vpn_tunnels | rejectattr('local_iface', 'equalto', vpn_primary_iface) + | rejectattr('local_addr', 'equalto', '%any') | list %} +{% if srcrt | length == 0 %} + : # single uplink - nothing to do +{% else %} +{% for t in srcrt %} + GW="{{ t.local_gateway }}" + [ -n "$GW" ] || GW=$(ip -4 route show default dev {{ t.local_iface }} 2>/dev/null | awk '{print $3; exit}') + if [ -n "$GW" ]; then + ip route replace default via "$GW" dev {{ t.local_iface }} table {{ t.srcrt_table }} + ip rule del from {{ t.local_addr }} lookup {{ t.srcrt_table }} 2>/dev/null || true + ip rule add from {{ t.local_addr }} lookup {{ t.srcrt_table }} + else + echo "WARN: no next hop found for {{ t.local_iface }} - tunnel {{ t.index }} may egress the wrong NIC" >&2 + fi +{% endfor %} +{% endif %} +} + +up_routes() { +{% if vpn_bgp_enabled | bool %} + # BGP owns the data prefixes. All this script installs is a host route per + # BGP peer, which is what pins each session to its own tunnel - without + # these the sessions never establish, because both children negotiate + # identical traffic selectors. +{% for t in vpn_tunnels if t.bgp_peer %} + ip route replace {{ t.bgp_peer }}/32 dev {{ t.xfrm_iface }} +{% endfor %} +{% else %} +{% if vpn_remote_subnet != "0.0.0.0/0" %} + ip route replace {{ vpn_remote_subnet }} {{ XFRM_NH }} +{% else %} + : # full tunnel - the mark table carries it, see up_mark_routing +{% endif %} +{% endif %} +} + +{% if vpn_transport == 'gre_fou' %} +up_gre() { + modprobe fou 2>/dev/null || true +{% if vpn_fou_disable_gro | bool %} + # GRO on the physical NIC coalesces inbound FOU/UDP packets. The + # coalesced packet cannot be re-encapsulated on egress and is dropped, + # which destroys TCP while leaving UDP and ICMP looking perfect. + # Correctness, not tuning - so it runs unconditionally here. + ethtool -K "$UPLINK" gro off 2>/dev/null || true +{% endif %} + # Purge EVERY gre-vpn* device, not just the ones we are about to create. + # Changing the gateway count renames the devices (gre-vpn <-> gre-vpn0,1..), + # and a leftover device with the same local/remote/key blocks the new one. + for _d in $(ip -o link show 2>/dev/null | awk -F': ' '{print $2}' \ + | sed 's/@.*//' | grep -E '^gre-vpn[0-9]*$'); do + ip link del "$_d" 2>/dev/null || true + ip tunnel del "$_d" 2>/dev/null || true + done +{% for g in gre_devs %} + ip fou del port {{ g.port }} 2>/dev/null || true +{% endfor %} + sleep 1 +{% for g in gre_devs %} + ip fou add port {{ g.port }} ipproto 47 + ip link add name {{ g.dev }} type gre local "$LOCAL_IP" key {{ g.key }} ttl 255 \ + encap fou encap-sport {{ 'auto' if vpn_fou_sport_auto | bool else g.port }} encap-dport {{ g.port }} + ip link set {{ g.dev }} mtu {{ vpn_gre_mtu }} + ip link set {{ g.dev }} up +{% endfor %} +} + +# Multipoint GRE has no signalling, so the neighbour table is pre-populated +# across the whole client CIDR. New VMs and K8s nodes are then covered without +# touching the gateway. +up_gre_neighbours() { + # One `ip` process for the whole CIDR, not one per host: a /20 is 4094 + # entries, and per-host spawning turns that into minutes. + python3 - <<'PYEOF' | ip -batch - +import ipaddress, sys +gw = "{{ vpn_local_gw_ip }}" +devs = [{% for g in gre_devs %}"{{ g.dev }}"{{ "," if not loop.last }}{% endfor %}] +net = ipaddress.ip_network("{{ vpn_client_cidr }}", strict=False) +n = 0 +for ip in net.hosts(): + s = str(ip) + if s == gw: + continue + for d in devs: + print(f"neigh replace {s} lladdr {s} dev {d} nud permanent") + n += 1 +print(f"populated {n} GRE neighbour entries for {net} " + f"across {len(devs)} device(s)", file=sys.stderr) +PYEOF +} + +# Traffic arriving from IPsec has to reach the overlay, not the main table. +up_overlay_routing() { +{% for t in vpn_tunnels %} + ip rule del iif {{ t.xfrm_iface }} lookup {{ vpn_rt_table }} 2>/dev/null || true + ip rule add iif {{ t.xfrm_iface }} lookup {{ vpn_rt_table }} +{% endfor %} + ip route replace {{ vpn_client_cidr }} {{ GRE_NH }} table {{ vpn_rt_table }} +} +{% endif %} + +{% if vpn_serves_clients | bool %} +# Client traffic may be destined anywhere (full tunnel), so mark it and route +# it through a dedicated table. The gateway's own default route is untouched. +up_mark_routing() { +{% if vpn_transport == 'gre_fou' %} +{% for g in gre_devs %} + iptables -t mangle -D PREROUTING -i {{ g.dev }} -j MARK --set-mark {{ vpn_fwmark }} 2>/dev/null || true + iptables -t mangle -A PREROUTING -i {{ g.dev }} -j MARK --set-mark {{ vpn_fwmark }} +{% endfor %} +{% else %} + # route transport: match client traffic on the uplink, but never traffic + # that stays inside the local subnet (that includes this gateway's own IP). + iptables -t mangle -D PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} ! -d {{ vpn_client_cidr }} -j MARK --set-mark {{ vpn_fwmark }} 2>/dev/null || true + iptables -t mangle -A PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} ! -d {{ vpn_client_cidr }} -j MARK --set-mark {{ vpn_fwmark }} +{% endif %} + ip rule del fwmark {{ vpn_fwmark }} lookup {{ vpn_mark_table }} 2>/dev/null || true + ip rule add fwmark {{ vpn_fwmark }} lookup {{ vpn_mark_table }} + ip route replace default {{ XFRM_NH }} table {{ vpn_mark_table }} +} +{% endif %} + +{% if vpn_mss_clamp | bool %} +up_mss_clamp() { + # Delete EVERY existing TCPMSS rule, not just the one we are about to add. + # Deleting only the exact rule leaves an older rule with a different MSS + # in place, and the stale one matches first - so a corrected value never + # takes effect. This bit us live: a stale --set-mss 1350 kept blackholing + # full-size segments after the value was fixed to 1320. + while iptables -t mangle -S FORWARD 2>/dev/null | grep -q -- '-j TCPMSS'; do + rule=$(iptables -t mangle -S FORWARD | grep -m1 -- '-j TCPMSS' | sed 's/^-A FORWARD //') + # shellcheck disable=SC2086 + iptables -t mangle -D FORWARD $rule 2>/dev/null || break + done + iptables -t mangle -A FORWARD -p tcp --syn -j TCPMSS {{ MSS_ARG }} +} +{% endif %} + +{% if vpn_disable_conntrack | bool %} +# Nothing here needs conntrack. NOTRACK removes its per-packet cost and, more +# importantly, enforces the statelessness that multi-gateway ECMP depends on: +# a reply may legitimately return through a DIFFERENT gateway. +up_notrack() { +{% if vpn_transport == 'route' %} + iptables -t raw -D PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} -j NOTRACK 2>/dev/null || true + iptables -t raw -A PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} -j NOTRACK +{% endif %} +{% for g in gre_devs %} + iptables -t raw -D PREROUTING -i {{ g.dev }} -j NOTRACK 2>/dev/null || true + iptables -t raw -A PREROUTING -i {{ g.dev }} -j NOTRACK +{% endfor %} +{% for t in vpn_tunnels %} + iptables -t raw -D PREROUTING -i {{ t.xfrm_iface }} -j NOTRACK 2>/dev/null || true + iptables -t raw -A PREROUTING -i {{ t.xfrm_iface }} -j NOTRACK +{% endfor %} +} +{% endif %} + +do_up() { + # The health timer remembers the last nexthop set it installed and only + # rewrites the route when that set changes. The full route is about to be + # reinstalled underneath it, so forget the memo: the first tick after an + # (re)deploy or service restart must reconcile against reality. + rm -f /run/vpn-ecmp-health.state 2>/dev/null || true + up_xfrm + up_source_routing + up_routes +{% if vpn_transport == 'gre_fou' %} + up_gre + up_gre_neighbours + up_overlay_routing +{% endif %} +{% if vpn_serves_clients | bool %} + up_mark_routing +{% endif %} +{% if vpn_mss_clamp | bool %} + up_mss_clamp +{% endif %} +{% if vpn_disable_conntrack | bool %} + up_notrack +{% endif %} +{% if vpn_perf_tuning | bool %} + [ -x /usr/local/sbin/vpn-perf-tuning.sh ] && /usr/local/sbin/vpn-perf-tuning.sh || true +{% endif %} + echo "vpn-network up: {{ vpn_tunnels | length }} tunnel(s), transport {{ vpn_transport }}" +} + +# =========================================================================== +# down +# =========================================================================== + +do_down() { +{% if vpn_mss_clamp | bool %} + while iptables -t mangle -S FORWARD 2>/dev/null | grep -q -- '-j TCPMSS'; do + rule=$(iptables -t mangle -S FORWARD | grep -m1 -- '-j TCPMSS' | sed 's/^-A FORWARD //') + # shellcheck disable=SC2086 + iptables -t mangle -D FORWARD $rule 2>/dev/null || break + done +{% endif %} +{% if vpn_disable_conntrack | bool %} +{% if vpn_transport == 'route' %} + iptables -t raw -D PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} -j NOTRACK 2>/dev/null || true +{% endif %} +{% for g in gre_devs %} + iptables -t raw -D PREROUTING -i {{ g.dev }} -j NOTRACK 2>/dev/null || true +{% endfor %} +{% for t in vpn_tunnels %} + iptables -t raw -D PREROUTING -i {{ t.xfrm_iface }} -j NOTRACK 2>/dev/null || true +{% endfor %} +{% endif %} +{% if vpn_serves_clients | bool %} +{% if vpn_transport == 'gre_fou' %} +{% for g in gre_devs %} + iptables -t mangle -D PREROUTING -i {{ g.dev }} -j MARK --set-mark {{ vpn_fwmark }} 2>/dev/null || true +{% endfor %} +{% else %} + iptables -t mangle -D PREROUTING -i "$UPLINK" -s {{ vpn_client_cidr }} ! -d {{ vpn_client_cidr }} -j MARK --set-mark {{ vpn_fwmark }} 2>/dev/null || true +{% endif %} + ip rule del fwmark {{ vpn_fwmark }} lookup {{ vpn_mark_table }} 2>/dev/null || true + ip route flush table {{ vpn_mark_table }} 2>/dev/null || true +{% endif %} +{% if vpn_transport == 'gre_fou' %} +{% for t in vpn_tunnels %} + ip rule del iif {{ t.xfrm_iface }} lookup {{ vpn_rt_table }} 2>/dev/null || true +{% endfor %} + ip route flush table {{ vpn_rt_table }} 2>/dev/null || true +{% for g in gre_devs %} + ip link del {{ g.dev }} 2>/dev/null || true + ip tunnel del {{ g.dev }} 2>/dev/null || true + ip fou del port {{ g.port }} 2>/dev/null || true +{% endfor %} +{% endif %} +{% for t in vpn_tunnels %} +{% if t.local_addr != '%any' and t.local_iface != vpn_primary_iface %} + ip rule del from {{ t.local_addr }} lookup {{ t.srcrt_table }} 2>/dev/null || true + ip route flush table {{ t.srcrt_table }} 2>/dev/null || true +{% endif %} + ip link del {{ t.xfrm_iface }} 2>/dev/null || true +{% endfor %} + echo "vpn-network down" +} + +case "${1:-up}" in + up) do_up ;; + down) do_down ;; + *) echo "usage: $0 {up|down}" >&2; exit 2 ;; +esac diff --git a/strongswan-ipsec/ansible/site.yml b/strongswan-ipsec/ansible/site.yml index 7447746..b44344c 100644 --- a/strongswan-ipsec/ansible/site.yml +++ b/strongswan-ipsec/ansible/site.yml @@ -1,11 +1,83 @@ --- -# 1. Configure gateways (IPsec + conditional FOU) -# 2. Configure Crusoe client VMs (FOU tunnel + route) -# Remote cloud VMs (Azure/GCP/AWS) need nothing. +# ============================================================================= +# Deploy the whole thing. +# +# ansible-playbook -i inventory.ini site.yml +# +# You edit two files: inventory.ini (who) and group_vars/all.yml (what). +# Everything else is already correct for a managed cloud peer. +# +# Four plays, in this order for a reason: +# +# 1. PREFLIGHT nothing is configured until every check passes on every +# host: the five required values, an apt-based OS with +# reachable mirrors, the kernel modules the datapath needs, a +# 1500-byte uplink, every client VM inside vpn_client_cidr, +# and UDP 500 + 4500 open in BOTH directions between the +# gateways in this run. any_errors_fatal, so one bad host +# stops the run before anything has been touched. Skipped for +# --tags teardown, which must work on a broken deployment. +# 2. KERNEL install a VAES-capable kernel if the running one predates +# 6.11, and reboot - but ONLY a host with no VPN on it yet. On +# a first deploy that is before any config exists, so nothing +# is interrupted. On a host that already carries tunnels the +# kernel is installed and the reboot is left to you (or to +# -e vpn_kernel_reboot_live=true). Never reboots a host that +# is already running the requested kernel. +# 3. GATEWAYS IPsec, XFRM interfaces, ECMP, firewall, host tuning. +# 4. CLIENTS routes from the Crusoe VMs into the overlay. +# +# Play 4 reads each gateway's resolved private IP from that gateway's facts, +# so plays 3 and 4 cannot be reordered or run in isolation. If you must limit +# to clients, set vpn_local_gw_ip per gateway in the inventory. +# +# Remote cloud VMs (Azure/GCP/AWS) need nothing - their cloud route tables +# already point at the VPN gateway. +# ============================================================================= + +- name: Preflight + hosts: vpn_gateways:vpn_clients + become: true + # One failure anywhere stops the whole run. A half-configured VPN is worse + # than an unconfigured one. + any_errors_fatal: true + # The UDP probe is three tasks (listen, send, check) that rely on every host + # finishing one before any host starts the next. That is the linear + # strategy's guarantee; pin it so a free-strategy ansible.cfg cannot break it. + strategy: linear + # Facts are gathered INSIDE the role, after the placeholder check. assert and + # set_fact are controller-side actions that need no connection, so a fresh + # clone with placeholder IPs gets the "fill in these five values" message + # instead of an SSH timeout. + gather_facts: false + tasks: + - name: Run every preflight check + ansible.builtin.include_role: + name: vpn_gateway + tasks_from: preflight.yml + # Runs for every path except teardown: a deployment you are removing may + # have exactly the broken firewall or dead mirror that preflight exists + # to catch, and must still be removable. + when: + - vpn_preflight | default(true) | bool + - "'teardown' not in (ansible_run_tags | default([]))" + tags: [preflight, setup, verify, kernel] + +- name: Ensure gateways have fast AES-GCM + hosts: vpn_gateways + become: true + any_errors_fatal: true + tasks: + - name: Install, and where safe boot, a VAES-capable kernel + ansible.builtin.include_role: + name: vpn_gateway + tasks_from: kernel.yml + tags: [kernel, setup] - name: Configure VPN gateways hosts: vpn_gateways become: true + any_errors_fatal: true roles: - vpn_gateway diff --git a/strongswan-ipsec/k8s/vpn-client.yaml b/strongswan-ipsec/k8s/vpn-client.yaml index 15b25b1..8d9646e 100644 --- a/strongswan-ipsec/k8s/vpn-client.yaml +++ b/strongswan-ipsec/k8s/vpn-client.yaml @@ -1,14 +1,19 @@ --- -# VPN Client DaemonSet — GRE-over-FOU tunnel from K8s nodes to the -# Crusoe VPN gateway. Same setup as the vpn_client Ansible role, but -# runs as a DaemonSet for managed K8s nodes where Ansible can't reach. +# VPN Client DaemonSet - configures K8s nodes to reach the remote site through +# one or more VPN gateways. Same job as the vpn_client Ansible role, for +# managed clusters where Ansible cannot reach the nodes. # -# The gateway's multipoint GRE tunnel already covers the entire node -# CIDR, so new nodes are handled automatically. +# Two transports, matching the Ansible role: +# gre_fou GRE-over-FOU overlay (default) +# route plain static routes via the gateway private IPs. Requires Crusoe +# port security to be DISABLED on every GATEWAY's vNIC. # -# IMPORTANT: Use a public image (like nicolaka/netshoot) so nodes can -# pull it before the VPN tunnel is up. Once the tunnel is established, -# nodes can pull private images from the datacenter registry. +# Traffic is ECMP'd across all gateways, hashed per flow. Nothing here is +# stateful, so a reply may come back through a different gateway - which is +# exactly what Azure does. See the README's "Rules that come with ECMP". +# +# IMPORTANT: use a public image (like nicolaka/netshoot) so nodes can pull it +# before the VPN is up. Once traffic flows, nodes can pull private images. # # Apply with: # kubectl apply -f vpn-client.yaml @@ -19,23 +24,70 @@ metadata: name: vpn-client-config namespace: kube-system data: - # VPN gateway private IP - GATEWAY_IP: "" - # GRE-over-FOU settings — must match the gateway's config + # Space-separated private IPs of the VPN gateways. One entry = today's + # behaviour. Several entries = ECMP across them. + GATEWAY_IPS: "" + + # gre_fou | route - must match the gateways + TRANSPORT: "gre_fou" + + # GRE-over-FOU settings - must match the gateways (vpn_gre_key, + # vpn_fou_port, vpn_gre_mtu in the vpn_gateway role). The gateway strips the + # GRE wrapper before encrypting, so the GRE MTU equals the 1400-byte tunnel + # MTU; it must never exceed it, nor the uplink MTU minus 36. GRE_KEY: "100" FOU_PORT: "9473" - GRE_MTU: "1360" - # Remote CIDRs to route through the VPN (space-separated) - # Split tunnel (datacenter only): + GRE_MTU: "1400" + # "true" makes the kernel hash the outer FOU source port per inner flow, so + # one node<->gateway pair stops being a single RSS flow. + FOU_SPORT_AUTO: "false" + # Extra FOU ports per gateway, for kernels without "encap-sport auto". + FOU_PORT_COUNT: "1" + + # route transport: MTU on the ECMP route. Azure's documented tunnel MTU. + ROUTE_MTU: "1400" + + # GRO MUST be off on the node's uplink when using gre_fou. Generic Receive + # Offload coalesces inbound FOU/UDP packets; the coalesced packet cannot be + # re-encapsulated and is dropped. UDP and ICMP are unaffected, so the tunnel + # looks perfectly healthy while TCP collapses. Measured on 8 vCPU VMs: + # 3.65 Mbps with GRO on, 4590 Mbps with it off. + DISABLE_GRO: "true" + + # Remote CIDRs to route through the VPN (space-separated). + # Split tunnel (remote site only): # REMOTE_CIDRS: "" - # Full tunnel (all internet via datacenter firewall): + # Full tunnel (all internet via the remote firewall): REMOTE_CIDRS: "0.0.0.0/1 128.0.0.0/1" # Intra-cluster traffic (node subnet, pod CIDR) is unaffected because # those routes are more specific (/20, /24) than /1. - # The datacenter gateway needs vpn_remote_subnet: 0.0.0.0/0 and a - # default route through the firewall for internet egress. - # Sync interval in seconds - SYNC_INTERVAL: "30" + + # --- node tuning --- + # THE most important one. 0 (kernel default) hashes src/dst IP only, so + # every flow takes ONE gateway and the others sit idle. The cluster looks + # healthy and performs like a single gateway. + MULTIPATH_HASH_POLICY: "1" + # route transport: skip nexthops whose ARP entry has gone unreachable. + MULTIPATH_USE_NEIGH: "1" + RP_FILTER: "2" + + # Long-fat-pipe TCP. At 50 ms RTT, 10 Gbps needs ~62 MB in flight, but the + # 6 MB default caps a single flow near 1 Gbps no matter how many tunnels + # exist underneath. Set TCP_TUNING to "false" to leave node TCP alone. + TCP_TUNING: "true" + TCP_RMEM: "4096 131072 134217728" + TCP_WMEM: "4096 65536 134217728" + RMEM_MAX: "268435456" + WMEM_MAX: "268435456" + # CUBIC collapses on a lossy internet path and does not recover. BBR needs fq. + CONGESTION_CONTROL: "bbr" + DEFAULT_QDISC: "fq" + + # Probe interval. Gateways that stop answering are dropped from the ECMP + # route; the gateway's own health timer stops answering ICMP when it has no + # established SA, so a live node with a dead tunnel drops out too. + SYNC_INTERVAL: "15" + PROBE_TIMEOUT: "1" --- apiVersion: apps/v1 @@ -85,71 +137,153 @@ spec: command: ["/bin/bash", "-c"] args: - | - set -euo pipefail + set -uo pipefail - setup_tunnel() { - local LOCAL_IP - LOCAL_IP=$(ip -4 route get 1.1.1.1 | awk '/src/{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}') + read -r -a GATEWAYS <<< "$GATEWAY_IPS" + NGW=${#GATEWAYS[@]} + NPORT="${FOU_PORT_COUNT:-1}" - # Skip if tunnel is already up - if ip link show gre-vpn 2>/dev/null | grep -q UP; then - return 0 + # Flat path list: one entry per gateway x FOU port. + PATH_DEV=(); PATH_GW=(); PATH_PORT=(); PATH_KEY=() + build_paths() { + local total=$(( NGW * NPORT )) n=0 g p + for g in "${GATEWAYS[@]}"; do + for (( p=0; p/dev/null || true + sysctl -qw "net.ipv4.conf.all.rp_filter=${RP_FILTER}" 2>/dev/null || true + sysctl -qw "net.ipv4.conf.default.rp_filter=${RP_FILTER}" 2>/dev/null || true + if [ "$TRANSPORT" = "route" ]; then + sysctl -qw "net.ipv4.fib_multipath_use_neigh=${MULTIPATH_USE_NEIGH}" 2>/dev/null || true + fi + if [ "${TCP_TUNING:-true}" = "true" ]; then + modprobe tcp_bbr 2>/dev/null || true + sysctl -qw "net.core.rmem_max=${RMEM_MAX}" 2>/dev/null || true + sysctl -qw "net.core.wmem_max=${WMEM_MAX}" 2>/dev/null || true + sysctl -qw "net.ipv4.tcp_rmem=${TCP_RMEM}" 2>/dev/null || true + sysctl -qw "net.ipv4.tcp_wmem=${TCP_WMEM}" 2>/dev/null || true + sysctl -qw "net.core.default_qdisc=${DEFAULT_QDISC}" 2>/dev/null || true + sysctl -qw "net.ipv4.tcp_congestion_control=${CONGESTION_CONTROL}" 2>/dev/null \ + || echo "note: ${CONGESTION_CONTROL} unavailable on this kernel" fi + } - # Clean stale state - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port "$FOU_PORT" 2>/dev/null || true + setup_tunnels() { + [ "$TRANSPORT" = "gre_fou" ] || return 0 + local LOCAL_IP i sport UPLINK + LOCAL_IP=$(ip -4 route get 1.1.1.1 | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1); exit}') + modprobe fou 2>/dev/null || true - echo "Creating GRE-over-FOU tunnel: $LOCAL_IP -> $GATEWAY_IP" + # See DISABLE_GRO above. This is correctness, not tuning: + # with GRO on, TCP through the overlay collapses by ~1250x. + if [ "${DISABLE_GRO:-true}" = "true" ]; then + UPLINK=$(ip -o -4 route show default | awk '{print $5; exit}') + if [ -n "${UPLINK:-}" ]; then + ethtool -K "$UPLINK" gro off 2>/dev/null || \ + echo "WARNING: could not disable GRO on $UPLINK - TCP through the overlay will be very slow" + fi + fi - modprobe fou - ip fou add port "$FOU_PORT" ipproto 47 - ip link add name gre-vpn type gre \ - local "$LOCAL_IP" remote "$GATEWAY_IP" key "$GRE_KEY" ttl 255 \ - encap fou encap-sport "$FOU_PORT" encap-dport "$FOU_PORT" - ip link set gre-vpn mtu "$GRE_MTU" - ip link set gre-vpn up + for (( i=0; i<${#PATH_DEV[@]}; i++ )); do + ip link show "${PATH_DEV[$i]}" 2>/dev/null | grep -q UP && continue + ip link del "${PATH_DEV[$i]}" 2>/dev/null || true + ip tunnel del "${PATH_DEV[$i]}" 2>/dev/null || true + ip fou del port "${PATH_PORT[$i]}" 2>/dev/null || true - # Allow FOU from gateway - iptables -C INPUT -p udp --dport "$FOU_PORT" -s "$GATEWAY_IP" -j ACCEPT 2>/dev/null || \ - iptables -I INPUT 1 -p udp --dport "$FOU_PORT" -s "$GATEWAY_IP" -j ACCEPT + ip fou add port "${PATH_PORT[$i]}" ipproto 47 2>/dev/null || true + sport="${PATH_PORT[$i]}" + [ "${FOU_SPORT_AUTO:-false}" = "true" ] && sport="auto" + echo "creating ${PATH_DEV[$i]}: $LOCAL_IP -> ${PATH_GW[$i]} (UDP ${PATH_PORT[$i]}, sport $sport)" + ip link add name "${PATH_DEV[$i]}" type gre \ + local "$LOCAL_IP" remote "${PATH_GW[$i]}" key "${PATH_KEY[$i]}" ttl 255 \ + encap fou encap-sport "$sport" encap-dport "${PATH_PORT[$i]}" + ip link set "${PATH_DEV[$i]}" mtu "$GRE_MTU" + ip link set "${PATH_DEV[$i]}" up - echo "GRE-over-FOU tunnel up" + iptables -C INPUT -p udp --dport "${PATH_PORT[$i]}" -s "${PATH_GW[$i]}" -j ACCEPT 2>/dev/null || \ + iptables -I INPUT 1 -p udp --dport "${PATH_PORT[$i]}" -s "${PATH_GW[$i]}" -j ACCEPT + done } + # Rebuild the ECMP route from the gateways that answer. Safe to + # re-hash: nothing here is stateful, so a flow that moves to + # another gateway keeps working. sync_routes() { + local i nh=() up=0 mtu=() + for (( i=0; i<${#PATH_DEV[@]}; i++ )); do + ping -c 1 -W "${PROBE_TIMEOUT:-1}" -n -q "${PATH_GW[$i]}" >/dev/null 2>&1 || continue + if [ "$TRANSPORT" = "route" ]; then + nh+=(nexthop via "${PATH_GW[$i]}" weight 1) + else + ip link show "${PATH_DEV[$i]}" 2>/dev/null | grep -q UP || continue + nh+=(nexthop dev "${PATH_DEV[$i]}" weight 1) + fi + up=$(( up + 1 )) + done + + if [ "$up" -eq 0 ]; then + echo "WARNING: no gateway answered - leaving routes alone" >&2 + return 0 + fi + [ "$TRANSPORT" = "route" ] && [ -n "${ROUTE_MTU:-}" ] && mtu=(mtu "$ROUTE_MTU") + + local desired="${nh[*]}" + if [ "$desired" != "${LAST_NH:-}" ]; then + echo "ECMP: $up/${#PATH_DEV[@]} path(s) up" + LAST_NH="$desired" + fi for cidr in $REMOTE_CIDRS; do - ip route replace "$cidr" dev gre-vpn 2>/dev/null || true + ip route replace "$cidr" "${nh[@]}" "${mtu[@]}" 2>/dev/null || true done } cleanup() { - echo "Cleaning up" + echo "cleaning up" for cidr in $REMOTE_CIDRS; do - ip route del "$cidr" dev gre-vpn 2>/dev/null || true + ip route del "$cidr" 2>/dev/null || true + done + local i + for (( i=0; i<${#PATH_DEV[@]}; i++ )); do + ip link del "${PATH_DEV[$i]}" 2>/dev/null || true + ip tunnel del "${PATH_DEV[$i]}" 2>/dev/null || true + ip fou del port "${PATH_PORT[$i]}" 2>/dev/null || true done - ip link del gre-vpn 2>/dev/null || true - ip tunnel del gre-vpn 2>/dev/null || true - ip fou del port "$FOU_PORT" 2>/dev/null || true exit 0 } - trap cleanup SIGTERM SIGINT - setup_tunnel + build_paths + apply_sysctls + setup_tunnels sync_routes - echo "VPN client running (sync every ${SYNC_INTERVAL}s)" - echo " Gateway: $GATEWAY_IP" - echo " Remote CIDRs: $REMOTE_CIDRS" + + echo "vpn-client running (transport=$TRANSPORT, sync every ${SYNC_INTERVAL}s)" + echo " gateways: $GATEWAY_IPS" + echo " paths: ${#PATH_DEV[@]}" + echo " remote: $REMOTE_CIDRS" + echo " hash policy: $(sysctl -n net.ipv4.fib_multipath_hash_policy 2>/dev/null)" + if [ "$TRANSPORT" = "gre_fou" ]; then + _u=$(ip -o -4 route show default | awk '{print $5; exit}') + echo " GRO on $_u: $(ethtool -k "$_u" 2>/dev/null | awk '/generic-receive-offload/{print $2}') (must be off)" + fi while true; do sleep "$SYNC_INTERVAL" & wait $! - if ! ip link show gre-vpn 2>/dev/null | grep -q UP; then - echo "Tunnel missing, recreating..." - setup_tunnel - fi + setup_tunnels sync_routes done resources: @@ -157,5 +291,5 @@ spec: cpu: 10m memory: 16Mi limits: - cpu: 50m - memory: 32Mi + cpu: 100m + memory: 64Mi