From 25d2ab8e8dcc1441872a459242807337637d710d Mon Sep 17 00:00:00 2001 From: Tanmoy Sarkar Date: Fri, 2 Oct 2026 22:57:57 +0530 Subject: [PATCH 1/5] feat(atlas): Number virtual machines per tenant --- atlas/atlas/core/mesh_address.py | 11 ++++------- atlas/atlas/core/test_mesh_address.py | 19 ++++++++----------- atlas/vm/SPEC.md | 2 +- .../virtual_machine/test_virtual_machine.py | 16 ++++++++-------- .../virtual_machine/virtual_machine.py | 4 ++-- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/atlas/atlas/core/mesh_address.py b/atlas/atlas/core/mesh_address.py index d6b0af32..cd87f410 100644 --- a/atlas/atlas/core/mesh_address.py +++ b/atlas/atlas/core/mesh_address.py @@ -14,9 +14,10 @@ MESH_PREFIX = 0xFDAA MESH_NETWORK = ipaddress.IPv6Network((MESH_PREFIX << 112, 16)) -MAXIMUM_VIRTUAL_MACHINE_NUMBER = 0xFFFFFFFFFFFFFFFF -# Atlas takes the last tenant-0 VM number, so no VM can use its address. -ATLAS_VIRTUAL_MACHINE_NUMBER = MAXIMUM_VIRTUAL_MACHINE_NUMBER +# A VM address is fdaa | region 16 | tenant 32 | padding 48 | VM number 16. The padding is zero. +MAXIMUM_VIRTUAL_MACHINE_NUMBER = 0xFFFF +# Atlas fills the padding and VM number, so no VM can use its address. +ATLAS_VIRTUAL_MACHINE_NUMBER = (1 << 64) - 1 WIREGUARD_PREFIX = 0xFDAB # The prefix and the region take the first 32 bits, so the low 96 bits of the UUID are the peer part. WIREGUARD_PEER_MASK = (1 << 96) - 1 @@ -42,10 +43,6 @@ def get_virtual_machine_mesh_address( frappe.throw( _("Virtual Machine number {0} is too large for a mesh address.").format(virtual_machine_number) ) - if virtual_machine.tenant_id == 0 and virtual_machine_number == ATLAS_VIRTUAL_MACHINE_NUMBER: - frappe.throw( - _("Virtual Machine number {0} is the Atlas mesh address.").format(virtual_machine_number) - ) address = ( (MESH_PREFIX << 112) | (region_id << 96) | (virtual_machine.tenant_id << 64) | virtual_machine_number diff --git a/atlas/atlas/core/test_mesh_address.py b/atlas/atlas/core/test_mesh_address.py index a0d7c380..9eca40be 100644 --- a/atlas/atlas/core/test_mesh_address.py +++ b/atlas/atlas/core/test_mesh_address.py @@ -17,21 +17,18 @@ def setUp(self) -> None: self.get_settings_patch.start() self.addCleanup(self.get_settings_patch.stop) - def test_a_wide_vm_number_fills_the_64_bit_field(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551615", tenant_id=7) + def test_the_largest_vm_number_leaves_the_padding_zero(self) -> None: + virtual_machine = frappe._dict(name="vm-7-65535", tenant_id=7) - address = get_virtual_machine_mesh_address(virtual_machine) + self.assertEqual(get_virtual_machine_mesh_address(virtual_machine), "fdaa:1:0:7::ffff") - self.assertEqual(address, "fdaa:1:0:7:ffff:ffff:ffff:ffff") + def test_the_vm_number_comes_after_the_tenant_in_the_name(self) -> None: + virtual_machine = frappe._dict(name="vm-42-0007", tenant_id=42) - def test_a_tenant_zero_vm_cannot_take_the_atlas_address(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551615", tenant_id=0) + self.assertEqual(get_virtual_machine_mesh_address(virtual_machine), "fdaa:1:0:2a::7") - with self.assertRaises(frappe.ValidationError): - get_virtual_machine_mesh_address(virtual_machine) - - def test_a_vm_number_above_64_bits_is_refused(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551616", tenant_id=7) + def test_a_vm_number_above_16_bits_is_refused(self) -> None: + virtual_machine = frappe._dict(name="vm-7-65536", tenant_id=7) with self.assertRaises(frappe.ValidationError): get_virtual_machine_mesh_address(virtual_machine) diff --git a/atlas/vm/SPEC.md b/atlas/vm/SPEC.md index 0fe3c577..fede07f8 100644 --- a/atlas/vm/SPEC.md +++ b/atlas/vm/SPEC.md @@ -21,7 +21,7 @@ This file identifies code owners and rules that a code change must preserve. The ## Rules to preserve -- The VM record name is the Metal VM ID from the `vm-.#######` series. A deleted name is never reused. +- The VM record name is the Metal VM ID from the `vm--.####` series, one counter for each tenant. The number after the last dash is the VM number in the mesh address, from 1 to 65,535 within the tenant. A deleted name is never reused. - `VirtualMachineService.create` commits a draft and any public address request before it calls Metal. An uncertain response keeps the draft. Only confirmed Metal absence removes it. - Metal is the authority for current host state. `Virtual Machine State` is a cache for lists and image checks. A failed Metal read must not appear as a guest state. - Placement checks capacity under a MariaDB named lock with READ COMMITTED isolation. Keep the lock until the draft or migration reservation commits. CPU can be oversubscribed. Memory and disk cannot. diff --git a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py index 3c153799..64625f15 100644 --- a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py @@ -359,16 +359,16 @@ def test_request_rejects_boolean_tenant_id(self) -> None: class TestVirtualMachineDocument(UnitTestCase): - def test_autoname_assigns_permanent_virtual_machine_id(self) -> None: - virtual_machine = frappe.new_doc("Virtual Machine") - - with patch.object( - virtual_machine_module, "make_autoname", return_value="vm-0000042" - ) as make_autoname: + def test_each_tenant_numbers_its_virtual_machines_from_one(self) -> None: + self.addCleanup(frappe.db.delete, "Series", {"name": ["in", ["vm-4000000001-", "vm-4000000002-"]]}) + names = [] + for tenant_id in (4_000_000_001, 4_000_000_001, 4_000_000_002): + virtual_machine = frappe.new_doc("Virtual Machine") + virtual_machine.tenant_id = tenant_id virtual_machine.autoname() + names.append(virtual_machine.name) - self.assertEqual(virtual_machine.name, "vm-0000042") - make_autoname.assert_called_once_with("vm-.#######", doc=virtual_machine) + self.assertEqual(names, ["vm-4000000001-0001", "vm-4000000001-0002", "vm-4000000002-0001"]) # New records have no Server, so virtual-field reads must skip Metal lookup. def test_new_document_reads_virtual_fields_without_a_server(self) -> None: diff --git a/atlas/vm/doctype/virtual_machine/virtual_machine.py b/atlas/vm/doctype/virtual_machine/virtual_machine.py index caeb6e5d..d8462d93 100644 --- a/atlas/vm/doctype/virtual_machine/virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/virtual_machine.py @@ -69,8 +69,8 @@ class VirtualMachine(Document): # end: auto-generated types def autoname(self) -> None: - """Assign a permanent virtual machine ID.""" - self.name = make_autoname("vm-.#######", doc=self) + """Assign a permanent virtual machine ID. Each tenant has its own counter.""" + self.name = make_autoname(f"vm-{self.tenant_id}-.####", doc=self) @request_cache def get_metal_vm_info(self) -> MetalVirtualMachine | None: From 44f5d103f5020949a8409d9f44b05f08ce81deb1 Mon Sep 17 00:00:00 2001 From: Tanmoy Sarkar Date: Fri, 2 Oct 2026 22:58:53 +0530 Subject: [PATCH 2/5] feat(ipv6-router): Map a 32-bit tenant and a 16-bit VM number --- atlas/service/SPEC.md | 2 +- atlas/service/core/ipv6_router/address.py | 10 +++---- .../service/core/ipv6_router/test_address.py | 29 +++++++------------ .../test_ipv6_router_server.py | 2 +- docs/networking/ipv6-router.md | 22 +++++++------- docs/networking/public-ips.md | 2 +- services/ipv6-router/bpf/address.h | 20 ++++++------- services/ipv6-router/setup.sh | 7 +++-- 8 files changed, 43 insertions(+), 51 deletions(-) diff --git a/atlas/service/SPEC.md b/atlas/service/SPEC.md index 0b3cac4c..57dc86a8 100644 --- a/atlas/service/SPEC.md +++ b/atlas/service/SPEC.md @@ -46,7 +46,7 @@ See the [HTTP proxy specification](../../services/http-proxy/SPEC.md). ## IPv6 Router Server -- The pool needs a prefix of `/84` or shorter, no gateway, and no allocations. +- The pool needs a prefix of `/80` or shorter, no gateway, and no allocations. - Installation fails if the eBPF program is not attached. - Archive is refused while tenant allocations use the pool. diff --git a/atlas/service/core/ipv6_router/address.py b/atlas/service/core/ipv6_router/address.py index c6304d9f..81f255d8 100644 --- a/atlas/service/core/ipv6_router/address.py +++ b/atlas/service/core/ipv6_router/address.py @@ -10,8 +10,8 @@ from atlas.vm.core.models import IPV6_INTERNET_DESTINATION # Must match bpf/address.h in services/ipv6-router. -TENANT_BITS = 24 -MACHINE_BITS = 20 +TENANT_BITS = 32 +MACHINE_BITS = 16 MAXIMUM_PREFIX_LENGTH = 128 - TENANT_BITS - MACHINE_BITS @@ -31,11 +31,9 @@ def get_routed_ipv6(prefix: str, mesh_address: str) -> str: tenant = (mesh >> 64) & 0xFFFFFFFF machine = mesh & 0xFFFFFFFFFFFFFFFF - if tenant >= 1 << TENANT_BITS or machine >= 1 << MACHINE_BITS: + if machine >= 1 << MACHINE_BITS: frappe.throw( - _("Mesh address {0} has a tenant or VM number that the routed IPv6 layout cannot hold.").format( - mesh_address - ), + _("Mesh address {0} has padding that the routed IPv6 layout cannot hold.").format(mesh_address), exc=AtlasUserError, ) diff --git a/atlas/service/core/ipv6_router/test_address.py b/atlas/service/core/ipv6_router/test_address.py index fbea0971..8bd3d87d 100644 --- a/atlas/service/core/ipv6_router/test_address.py +++ b/atlas/service/core/ipv6_router/test_address.py @@ -7,31 +7,24 @@ class TestRoutedIPv6(UnitTestCase): - def test_the_tenant_and_vm_fill_the_low_44_bits(self) -> None: - self.assertEqual( - get_routed_ipv6("2001:db8:1:2:3::/80", "fdaa:1:0:abcd::5"), - "2001:db8:1:2:3:a:bcd0:5", - ) + def test_the_tenant_and_vm_fill_the_low_48_bits(self) -> None: + self.assertEqual(get_routed_ipv6("2001:db8:1:2:3::/80", "fdaa:1:0:2a::7"), "2001:db8:1:2:3:0:2a:7") - def test_a_shorter_prefix_keeps_zero_reserved_bits(self) -> None: - self.assertEqual(get_routed_ipv6("2001:db8:1::/48", "fdaa:1:0:2::3"), "2001:db8:1::20:3") + def test_a_shorter_prefix_keeps_the_bits_above_the_host_part_zero(self) -> None: + self.assertEqual(get_routed_ipv6("2001:db8:1::/48", "fdaa:1:0:2::3"), "2001:db8:1::2:3") def test_the_largest_tenant_and_vm_fit(self) -> None: self.assertEqual( - get_routed_ipv6("2001:db8::/64", "fdaa:1:ff:ffff::f:ffff"), "2001:db8::fff:ffff:ffff" + get_routed_ipv6("2001:db8::/64", "fdaa:1:ffff:ffff::ffff"), "2001:db8::ffff:ffff:ffff" ) - def test_a_tenant_that_needs_more_than_24_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "cannot hold"): - get_routed_ipv6("2001:db8::/64", "fdaa:1:100:0::1") - - def test_a_vm_number_that_needs_more_than_20_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "cannot hold"): - get_routed_ipv6("2001:db8::/64", "fdaa:1:0:1::10:0") + def test_an_address_with_padding_is_refused(self) -> None: + with self.assertRaisesRegex(frappe.ValidationError, "padding"): + get_routed_ipv6("2001:db8::/64", "fdaa:1:0:1::1:0") - def test_a_block_smaller_than_84_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "/84"): - get_routed_ipv6("2001:db8::/96", "fdaa:1:0:1::1") + def test_a_block_smaller_than_80_bits_is_refused(self) -> None: + with self.assertRaisesRegex(frappe.ValidationError, "/80"): + get_routed_ipv6("2001:db8::/84", "fdaa:1:0:1::1") def test_a_noncanonical_block_is_refused(self) -> None: with self.assertRaisesRegex(frappe.ValidationError, "canonical IPv6 prefix"): diff --git a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py index e77885c3..513c2cd8 100644 --- a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py +++ b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py @@ -48,7 +48,7 @@ def test_a_pool_with_allocations_is_refused(self) -> None: self.validate(public_ip_pool(), has_allocations=True) def test_a_small_pool_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "/84"): + with self.assertRaisesRegex(frappe.ValidationError, "/80"): self.validate(public_ip_pool(prefix="2001:db8::/96")) diff --git a/docs/networking/ipv6-router.md b/docs/networking/ipv6-router.md index cc84b41c..b78c4efd 100644 --- a/docs/networking/ipv6-router.md +++ b/docs/networking/ipv6-router.md @@ -18,13 +18,13 @@ The VM can move while its mesh and public addresses stay the same. WG Mesh finds ## How the address maps ```text -public = | reserved, all zero | tenant 24 bits | VM 20 bits -private = fdaa | region 16 bits | tenant 32 bits | VM 64 bits +public = | tenant 32 bits | VM 16 bits +private = fdaa | region 16 bits | tenant 32 bits | padding 48 bits | VM 16 bits ``` -For block `2001:db8:1:2:3::/80`, mesh address `fdaa:1:0:abcd::5` maps to `2001:db8:1:2:3:a:bcd0:5`. +For block `2001:db8:1:2:3::/80`, mesh address `fdaa:1:0:2a::7` maps to `2001:db8:1:2:3:0:2a:7`. The tenant and the VM number keep their own hextets. -The block must be inside `2000::/3`, with a prefix from `/3` to `/84`. A VM whose tenant ID is 2^24 or more, or whose VM number is 2^20 or more, gets no public address. +The block must be inside `2000::/3`, with a prefix from `/3` to `/80`. In a shorter block, the bits between the block and the host part are zero. A mesh address with padding, such as the Atlas address, has no public address. ::: details Try the IPv6 address mapping in Python @@ -34,21 +34,21 @@ This example derives the public address and recovers the tenant and VM number fr from ipaddress import IPv6Address, IPv6Network block = IPv6Network("2001:db8:1:2:3::/80") -mesh = IPv6Address("fdaa:1:0:abcd::5") +mesh = IPv6Address("fdaa:1:0:2a::7") region = 1 tenant = (int(mesh) >> 64) & 0xFFFFFFFF vm = int(mesh) & 0xFFFFFFFFFFFFFFFF -assert tenant < 1 << 24 and vm < 1 << 20 +assert vm < 1 << 16 -public = IPv6Address(int(block.network_address) | (tenant << 20) | vm) +public = IPv6Address(int(block.network_address) | (tenant << 16) | vm) suffix = int(public) - int(block.network_address) -decoded_tenant, decoded_vm = divmod(suffix, 1 << 20) +decoded_tenant, decoded_vm = divmod(suffix, 1 << 16) decoded_mesh = IPv6Address( (0xFDAA << 112) | (region << 96) | (decoded_tenant << 64) | decoded_vm ) print(public, decoded_mesh) -# 2001:db8:1:2:3:a:bcd0:5 fdaa:1:0:abcd::5 +# 2001:db8:1:2:3:0:2a:7 fdaa:1:0:2a::7 ``` ::: @@ -61,8 +61,8 @@ The [gateway packet example](wg-mesh/gateways.md#how-the-packet-changes) shows t A `tc` eBPF program on the router's `eth0` ingress rewrites the address and fixes the checksum. For an ICMPv6 error, it also rewrites the address inside the quoted packet. It drops: -- A packet to a block address with a reserved bit set. -- A mesh source from another region, or a tenant or VM number that does not fit the public layout. +- A packet to a block address with a set bit between the block and the host part. +- A mesh source from another region, or a mesh source with padding. - A translated packet that is not TCP, UDP, ICMPv6, or a fragment of one of them. ## Block unassigned traffic diff --git a/docs/networking/public-ips.md b/docs/networking/public-ips.md index 836a0baa..9e5d659e 100644 --- a/docs/networking/public-ips.md +++ b/docs/networking/public-ips.md @@ -47,7 +47,7 @@ The router receives the full prefix. Each tenant VM gets a derived `/128`, so At Requirements: -- Prefix length `/84` or shorter. AWS `/80` and Scaleway `/64` qualify. +- Prefix length `/80` or shorter. AWS `/80` and Scaleway `/64` qualify. - No allocations in the pool before router creation. - An Active router before routed automatic assignment. diff --git a/services/ipv6-router/bpf/address.h b/services/ipv6-router/bpf/address.h index cb8619ae..c24be02d 100644 --- a/services/ipv6-router/bpf/address.h +++ b/services/ipv6-router/bpf/address.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: AGPL-3.0 */ /* The address layouts and the stateless mapping between them. * - * public = prefix | reserved, all zero | tenant 24 bits | virtual machine 20 bits - * private = fdaa | region 16 bits | tenant 32 bits | virtual machine 64 bits + * public = prefix, at most /80 | tenant 32 bits | virtual machine 16 bits + * private = fdaa | region 16 bits | tenant 32 bits | padding 48 bits, zero | virtual machine 16 bits */ #ifndef ATLAS_IPV6_ROUTER_ADDRESS_H #define ATLAS_IPV6_ROUTER_ADDRESS_H @@ -18,9 +18,9 @@ #define MESH_PREFIX 0xfdaaULL #define MESH_REGION ((MESH_PREFIX << 16) | REGION_ID) -#define TENANT_BITS 24 -#define MACHINE_BITS 20 -#define TENANT_LIMIT (1ULL << TENANT_BITS) +#define TENANT_BITS 32 +#define MACHINE_BITS 16 +#define TENANT_MASK ((1ULL << TENANT_BITS) - 1) #define MACHINE_LIMIT (1ULL << MACHINE_BITS) #define HOST_MASK ((1ULL << (TENANT_BITS + MACHINE_BITS)) - 1) @@ -68,7 +68,7 @@ static __always_inline bool is_in_block(const struct address *address) return (address->high & PUBLIC_HIGH_MASK) == PUBLIC_HIGH && (address->low & PUBLIC_LOW_MASK) == PUBLIC_LOW; } -/* A block address maps to a virtual machine only when its reserved bits are zero. */ +/* A block address maps to a virtual machine only when the bits between the block and the host part are zero. */ static __always_inline bool is_mapped_public(const struct address *address) { return address->high == PUBLIC_HIGH && (address->low & ~HOST_MASK) == PUBLIC_LOW; @@ -79,17 +79,17 @@ static __always_inline struct address to_private(const struct address *public) { struct address private; - private.high = (MESH_REGION << 32) | ((public->low >> MACHINE_BITS) & (TENANT_LIMIT - 1)); + private.high = (MESH_REGION << 32) | ((public->low >> MACHINE_BITS) & TENANT_MASK); private.low = public->low & (MACHINE_LIMIT - 1); return private; } -/* Returns false for another region, or a tenant or machine number that the public layout cannot hold. */ +/* Returns false for another region, or an address with padding, such as the Atlas address. */ static __always_inline bool to_public(const struct address *private, struct address *public) { - __u64 tenant = private->high & 0xffffffffULL; + __u64 tenant = private->high & TENANT_MASK; - if ((private->high >> 32) != MESH_REGION || tenant >= TENANT_LIMIT || private->low >= MACHINE_LIMIT) + if ((private->high >> 32) != MESH_REGION || private->low >= MACHINE_LIMIT) return false; public->high = PUBLIC_HIGH; diff --git a/services/ipv6-router/setup.sh b/services/ipv6-router/setup.sh index ba214f28..92afc7d3 100755 --- a/services/ipv6-router/setup.sh +++ b/services/ipv6-router/setup.sh @@ -34,7 +34,8 @@ printf '%s\n' "${modules[@]}" > /etc/modules-load.d/atlas-ipv6-router.conf step "write the configuration" install -d -m 0755 "$state_dir" -# The public host part needs 44 bits, so the prefix can be at most /84. + +# The public host part needs 48 bits, so the prefix can be at most /80. read -r mesh_region public_prefix < <(python3 - "$REGION_ID" "$PUBLIC_IPV6_PREFIX" "$state_dir/config.h" <<'PYTHON' import ipaddress import sys @@ -51,8 +52,8 @@ if not 0 <= region_id <= 0xFFFF: sys.exit(f"REGION_ID {region_id} is not a 16-bit number") if not network.subnet_of(ipaddress.IPv6Network("2000::/3")): sys.exit(f"PUBLIC_IPV6_PREFIX {network} is outside 2000::/3") -if network.prefixlen > 84: - sys.exit(f"PUBLIC_IPV6_PREFIX {network} is longer than /84") +if network.prefixlen > 80: + sys.exit(f"PUBLIC_IPV6_PREFIX {network} is longer than /80") words = { "PUBLIC_HIGH": int(network.network_address) >> 64, From 742b4c903562c86818103ffd4995d2b80d004d1f Mon Sep 17 00:00:00 2001 From: Tanmoy Sarkar Date: Fri, 2 Oct 2026 22:59:18 +0530 Subject: [PATCH 3/5] feat(wireguard-gateway): Put the peer number in the last hextet --- docs/networking/wireguard-gateway.md | 6 ++++-- services/wg-gateway/bpf/gateway.c | 4 ++-- services/wg-gateway/daemon/gatewayd/main.py | 2 +- services/wg-gateway/daemon/gatewayd/peers.py | 20 +++++++++---------- .../wg-gateway/daemon/tests/test_peers.py | 14 +++++++++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/docs/networking/wireguard-gateway.md b/docs/networking/wireguard-gateway.md index 7aafe2dd..399164f3 100644 --- a/docs/networking/wireguard-gateway.md +++ b/docs/networking/wireguard-gateway.md @@ -26,11 +26,13 @@ The answer holds the WireGuard settings of the device: | Field | Example | Use | | --- | --- | --- | -| `address` | `fdac:1:2:0:2a:0:9:0/128` | Interface address of the device. | +| `address` | `fdac:1:2:0:2a::9/128` | Interface address of the device. | | `allowed_ips` | `["fdaa:1:0:2a::/64"]` | The VM addresses of the tenant. | | `endpoint` | `wireguard-002.par-1.example.com:51820` | The node that serves the device. | | `public_key` | `` | Public key of that node. | +The `tenant_id` is from 1 to 4,294,967,295, and the `client_id` is from 1 to 65,535. + A repeated request with the same key returns the same settings. A new device goes to the node with the fewest devices. `DELETE /v1/peers` with `{"tenant_id": 42, "client_id": 9}` removes a device, and `GET /v1/peers` lists every device with its `node_id`. `PUT /v1/peers` replaces the complete table, for example to restore it from Central's records. Send each device's `node_id` from the list, because the device address carries the node number. A device without `node_id` goes to the node with the fewest devices. @@ -50,7 +52,7 @@ The gateway eBPF filters drop a packet unless the device and the VM belong to th ## Addresses ```text -fdac | region 16 | node 16 | tenant 32 | client 32 | zero 16 +fdac | region 16 | node 16 | tenant 32 | padding 32 | client 16 ``` Each node owns one `fdac::::/48`. Atlas reserves `fdaa::/16`, `fdab::/16`, and `fdac::/16`. A customer network that uses one of these ranges conflicts with the routes of its devices, so it must use another range. diff --git a/services/wg-gateway/bpf/gateway.c b/services/wg-gateway/bpf/gateway.c index b4e40755..e9b701df 100644 --- a/services/wg-gateway/bpf/gateway.c +++ b/services/wg-gateway/bpf/gateway.c @@ -16,8 +16,8 @@ static __always_inline int valid_client(const struct in6_addr *client, const str client->s6_addr16[3] == vm->s6_addr16[2] && client->s6_addr16[4] == vm->s6_addr16[3] && (client->s6_addr16[3] || client->s6_addr16[4]) && - (client->s6_addr16[5] || client->s6_addr16[6]) && - client->s6_addr16[7] == 0; + client->s6_addr16[5] == 0 && client->s6_addr16[6] == 0 && + client->s6_addr16[7]; } SEC("tc/client") diff --git a/services/wg-gateway/daemon/gatewayd/main.py b/services/wg-gateway/daemon/gatewayd/main.py index 17d25f6a..bb2a95e5 100644 --- a/services/wg-gateway/daemon/gatewayd/main.py +++ b/services/wg-gateway/daemon/gatewayd/main.py @@ -70,7 +70,7 @@ class PeerSettings(BaseModel): tenant_id: int = Field(examples=[42], description="Tenant of the peer.") client_id: int = Field(examples=[9], description="Peer number within the tenant.") - address: str = Field(examples=["fdac:1:2:0:2a:0:9:0/128"], description="Interface address of the peer.") + address: str = Field(examples=["fdac:1:2:0:2a::9/128"], description="Interface address of the peer.") allowed_ips: list[str] = Field( examples=[["fdaa:1:0:2a::/64"]], description="The VM addresses of the tenant. The peer puts them in its own AllowedIPs.", diff --git a/services/wg-gateway/daemon/gatewayd/peers.py b/services/wg-gateway/daemon/gatewayd/peers.py index 11bf0566..2e63f35c 100644 --- a/services/wg-gateway/daemon/gatewayd/peers.py +++ b/services/wg-gateway/daemon/gatewayd/peers.py @@ -14,7 +14,8 @@ PEERS_KIND = "peers" CLIENT_PREFIX = 0xFDAC MESH_PREFIX = 0xFDAA -FIELD_LIMIT = (1 << 32) - 1 +TENANT_LIMIT = (1 << 32) - 1 +CLIENT_LIMIT = (1 << 16) - 1 PUBLIC_KEY_PATTERN = re.compile(r"[A-Za-z0-9+/]{43}=") NODE_ID_PATTERN = re.compile(r"wireguard-[0-9]{3,5}") INTERFACE = "wg0" @@ -28,9 +29,12 @@ def get_identity(tenant_id: int, client_id: int) -> str: def validate_device(tenant_id: int, client_id: int, public_key: str) -> None: """Reject a device that the address layout or WireGuard cannot hold.""" - for label, value in (("tenant_id", tenant_id), ("client_id", client_id)): - if not 1 <= value <= FIELD_LIMIT: - raise HTTPException(status_code=400, detail=f"{label} must be from 1 to {FIELD_LIMIT}") + for label, value, limit in ( + ("tenant_id", tenant_id, TENANT_LIMIT), + ("client_id", client_id, CLIENT_LIMIT), + ): + if not 1 <= value <= limit: + raise HTTPException(status_code=400, detail=f"{label} must be from 1 to {limit}") if not PUBLIC_KEY_PATTERN.fullmatch(public_key): raise HTTPException(status_code=400, detail="public_key must be a 44-character WireGuard key") @@ -179,13 +183,7 @@ def get_credentials(self, identity: str, peer: dict[str, str]) -> dict[str, obje # The node number is part of the record name, so an archived node still gives the address. gateway_id = int(peer["node_id"].rsplit("-", 1)[-1]) region = self.config.region_id - address = ( - (CLIENT_PREFIX << 112) - | (region << 96) - | (gateway_id << 80) - | (tenant_id << 48) - | (client_id << 16) - ) + address = (CLIENT_PREFIX << 112) | (region << 96) | (gateway_id << 80) | (tenant_id << 48) | client_id tenant_network = (MESH_PREFIX << 112) | (region << 96) | (tenant_id << 64) return { "tenant_id": tenant_id, diff --git a/services/wg-gateway/daemon/tests/test_peers.py b/services/wg-gateway/daemon/tests/test_peers.py index f83cfaf7..5aecfc7d 100644 --- a/services/wg-gateway/daemon/tests/test_peers.py +++ b/services/wg-gateway/daemon/tests/test_peers.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from gatewayd.config import ConfigError, GatewayConfig, Node, load -from gatewayd.peers import PEERS_KIND, PeerState +from gatewayd.peers import PEERS_KIND, PeerState, validate_device KEY_A = "A" * 43 + "=" KEY_B = "B" * 43 + "=" @@ -50,7 +50,7 @@ def test_credentials_carry_the_node_tenant_and_client_in_the_address(tmp_path): credentials = register(peers, {}, "42:9", KEY_A, "wireguard-007") - assert credentials["address"] == "fdac:2:7:0:2a:0:9:0/128" + assert credentials["address"] == "fdac:2:7:0:2a::9/128" assert credentials["allowed_ips"] == ["fdaa:2:0:2a::/64"] assert credentials["endpoint"] == "wireguard-007.par-1.example.com:51821" assert credentials["public_key"] == "node-7-public" @@ -103,7 +103,7 @@ def test_only_this_node_applies_its_devices(tmp_path): assert peers.applied.count("[Peer]") == 1 assert f"PublicKey = {KEY_A}" in peers.applied - assert "AllowedIPs = fdac:2:1:0:1:0:1:0/128" in peers.applied + assert "AllowedIPs = fdac:2:1:0:1::1/128" in peers.applied def test_a_repeated_registration_keeps_the_first_node(tmp_path): @@ -162,7 +162,7 @@ def test_a_device_of_an_archived_node_is_kept_without_an_endpoint(tmp_path): asyncio.run(peers.apply(state, Mutation(kind=PEERS_KIND, action="replace", value=table))) credentials = peers.get_credentials("1:1", state[PEERS_KIND]["1:1"]) - assert credentials["address"] == "fdac:2:2:0:1:0:1:0/128" + assert credentials["address"] == "fdac:2:2:0:1::1/128" assert (credentials["endpoint"], credentials["public_key"]) == ("", "") with pytest.raises(HTTPException): register(peers, state, "1:2", KEY_B, "wireguard-002") @@ -224,3 +224,9 @@ def apply_wireguard(self, state): with pytest.raises(HTTPException): register(peers, state, "1:1", KEY_A, "wireguard-001") assert state == {PEERS_KIND: {}} + + +def test_a_client_number_must_fit_one_hextet(): + validate_device(1, 65535, KEY_A) + with pytest.raises(HTTPException): + validate_device(1, 65536, KEY_A) From 74d0b64dbbaef95764a121ae20a484ba9c0f03b7 Mon Sep 17 00:00:00 2001 From: Tanmoy Sarkar Date: Fri, 2 Oct 2026 22:59:39 +0530 Subject: [PATCH 4/5] feat(http-proxy): Decode 48-bit auto-proxy labels --- docs/networking/http-proxy/openresty.md | 4 ++-- .../http-proxy/nginx/lua/http/auto_proxy.lua | 17 ++++------------- services/http-proxy/tests/test_proxy.py | 6 +++--- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/docs/networking/http-proxy/openresty.md b/docs/networking/http-proxy/openresty.md index 8c364b9d..104e2182 100644 --- a/docs/networking/http-proxy/openresty.md +++ b/docs/networking/http-proxy/openresty.md @@ -31,7 +31,7 @@ The wildcard certificate covers site HTTPS traffic. An address of `-` returns `5 An auto-proxy name sends HTTP to a VM without a saved site-map entry. The hostname uses a configured prefix, such as `site-` or `*-vm-`, followed by a base-36 label. OpenResty sends the request to the decoded VM mesh address on port `80`. -Atlas puts the 64-bit VM number above the 32-bit tenant ID: `value = (vm << 32) | tenant`. The VM number is the numeric suffix of its Atlas ID, so `VM-00011` uses `11`. Atlas writes the 96-bit value in lowercase base 36. The regional mesh prefix supplies `fdaa:` when OpenResty decodes the label. +Atlas puts the 16-bit VM number above the 32-bit tenant ID: `value = (vm << 32) | tenant`. The VM number is the numeric suffix of its Atlas ID, so `vm-2-0011` uses `11`. Atlas writes the 48-bit value in lowercase base 36, in at most 10 characters. The regional mesh prefix supplies `fdaa:` when OpenResty decodes the label. ::: details Try the base-36 mapping in Python @@ -62,7 +62,7 @@ print(name, mesh) # site-lpc8lqa.example.com fdaa:1:0:2::b ::: -The proxy accepts only configured host prefixes and a canonical lowercase label. The label has at most 19 digits and no leading zero. Values larger than 96 bits and unsupported prefixes do not route. The control daemon reserves matching site keys so a stored route cannot override an auto-proxy name. +The proxy accepts only configured host prefixes and a canonical lowercase label. The label has at most 10 digits and no leading zero. Values larger than 48 bits and unsupported prefixes do not route. The control daemon reserves matching site keys so a stored route cannot override an auto-proxy name. ::: info Static route, live destination The name encodes tenant and VM identity, not a host location. WG Mesh finds the VM's current Metal host after a move. An auto-proxy name alone does not grant access through tenant or guest firewall rules. diff --git a/services/http-proxy/nginx/lua/http/auto_proxy.lua b/services/http-proxy/nginx/lua/http/auto_proxy.lua index 3802e03e..7390c0f3 100644 --- a/services/http-proxy/nginx/lua/http/auto_proxy.lua +++ b/services/http-proxy/nginx/lua/http/auto_proxy.lua @@ -13,7 +13,7 @@ local function digit_value(byte) end local function append_digit(address_parts, digit) - -- Keep the 96-bit value in six 16-bit address parts. + -- Keep the 48-bit value in three 16-bit address parts. local carry = digit for position = #address_parts, 1, -1 do local value = address_parts[position] * BASE + carry @@ -24,7 +24,7 @@ local function append_digit(address_parts, digit) end local function decode_token(token) - local address_parts = { 0, 0, 0, 0, 0, 0 } + local address_parts = { 0, 0, 0 } for position = 1, #token do local digit = digit_value(token:byte(position)) if not digit or not append_digit(address_parts, digit) then @@ -56,17 +56,8 @@ local function is_allowed_prefix(host_prefix, host_prefixes) end local function format_address(address_prefix, address_parts) - -- IPv6 stores tenant parts before VM parts. - return string.format( - "%s:%x:%x:%x:%x:%x:%x", - address_prefix, - address_parts[5], - address_parts[6], - address_parts[1], - address_parts[2], - address_parts[3], - address_parts[4] - ) + -- The label puts the VM number before the tenant, and the address puts it after. + return string.format("%s:%x:%x::%x", address_prefix, address_parts[2], address_parts[3], address_parts[1]) end function auto_proxy.get_virtual_machine_address(subdomain, address_prefix, host_prefixes) diff --git a/services/http-proxy/tests/test_proxy.py b/services/http-proxy/tests/test_proxy.py index 87281cdb..5f53b567 100644 --- a/services/http-proxy/tests/test_proxy.py +++ b/services/http-proxy/tests/test_proxy.py @@ -192,10 +192,10 @@ def test_a_map_entry_never_overrides_an_auto_proxy_host(): [ # A padded token is not the label of the VM it decodes to. "site-02avaxbsw", - # 20 digits cannot hold a 96-bit value. - "site-zzzzzzzzzzzzzzzzzzzz", + # 11 digits cannot hold a 48-bit value. + "site-zzzzzzzzzzz", # One above the widest tenant and VM number pair. - "site-7oiylpimjg5u2ca1yps", + "site-2rrvthnxts", # The prefix is not configured. "shop-2avaxbsw", # A bare label carries no prefix. From baed4d93eaefb394749a37359e642f464e306080 Mon Sep 17 00:00:00 2001 From: Tanmoy Sarkar Date: Fri, 2 Oct 2026 22:59:51 +0530 Subject: [PATCH 5/5] docs(atlas): Describe the IPv6 address formats --- .vitepress/sidebar.mts | 1 + atlas/vm/SPEC.md | 2 +- docs/networking/address-formats.md | 71 ++++++++++++++++++++++++++++ docs/networking/index.md | 4 +- docs/networking/ipv6-router.md | 2 +- docs/networking/wg-mesh/index.md | 2 +- docs/networking/wireguard-gateway.md | 2 +- docs/reference/glossary.md | 2 +- services/wg-mesh/bpf/mesh.h | 2 +- 9 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 docs/networking/address-formats.md diff --git a/.vitepress/sidebar.mts b/.vitepress/sidebar.mts index 62e2ed1f..d156fe96 100644 --- a/.vitepress/sidebar.mts +++ b/.vitepress/sidebar.mts @@ -56,6 +56,7 @@ const handbook: DefaultTheme.SidebarItem[] = [ { text: 'Benchmarks', link: '/docs/networking/wg-mesh/benchmarks' }, ], }, + { text: 'Address formats', link: '/docs/networking/address-formats' }, { text: 'Host networking', link: '/docs/networking/host-networking' }, { text: 'Public IPs', link: '/docs/networking/public-ips' }, { diff --git a/atlas/vm/SPEC.md b/atlas/vm/SPEC.md index fede07f8..fdf2f8ab 100644 --- a/atlas/vm/SPEC.md +++ b/atlas/vm/SPEC.md @@ -21,7 +21,7 @@ This file identifies code owners and rules that a code change must preserve. The ## Rules to preserve -- The VM record name is the Metal VM ID from the `vm--.####` series, one counter for each tenant. The number after the last dash is the VM number in the mesh address, from 1 to 65,535 within the tenant. A deleted name is never reused. +- The VM record name is the Metal VM ID from the `vm--.####` series, one counter for each tenant. The number after the last dash is the VM number in the mesh address, from 1 to 65,535 within the tenant. See [address formats](../../docs/networking/address-formats.md). A deleted name is never reused. - `VirtualMachineService.create` commits a draft and any public address request before it calls Metal. An uncertain response keeps the draft. Only confirmed Metal absence removes it. - Metal is the authority for current host state. `Virtual Machine State` is a cache for lists and image checks. A failed Metal read must not appear as a guest state. - Placement checks capacity under a MariaDB named lock with READ COMMITTED isolation. Keep the lock until the draft or migration reservation commits. CPU can be oversubscribed. Memory and disk cannot. diff --git a/docs/networking/address-formats.md b/docs/networking/address-formats.md new file mode 100644 index 00000000..574fd247 --- /dev/null +++ b/docs/networking/address-formats.md @@ -0,0 +1,71 @@ +# Address formats + +Atlas derives each private IPv6 address from IDs. No service looks up an address in a table. This page is the reference for each layout. + +## Rules for every layout + +Each field starts on a 16-bit boundary, so you can read the IDs from the address. The first hextet gives the address type, and the second hextet gives the region. + +The tenant ID has 32 bits. A VM number and a WireGuard peer number have 16 bits each and use the last hextet. + +Padding fields are zero. They hold space for later fields. Do not use them. + +| Prefix | Use | +| --- | --- | +| `fdaa::/16` | VM mesh addresses and the Atlas address | +| `fdab::/16` | Host `wg0` addresses | +| `fdac::/16` | WireGuard gateway peers | + +## VM mesh address + +```text +fdaa : region : tenant (32 bits) : padding (48 bits) : VM number +``` + +Example: `vm-42-0007` in region `1` is `fdaa:1:0:2a::7`. + +The VM name is `vm--`. Each tenant has its own counter, so a VM number is unique only within its tenant. Each tenant has the `/64` `fdaa::::/64`. A tenant can have VM numbers from 1 to 65,535. Atlas refuses a larger number. + +## Atlas address + +```text +fdaa : region : 0 : 0 : ffff : ffff : ffff : ffff +``` + +The Atlas address is in tenant 0 and has all padding bits set. No VM can have this address. + +## Host wg0 address + +```text +fdab : region : low 96 bits of the host UUID +``` + +Example: `fdab:1:7f90:766a:8d2c:2f76:8d76:2045`. This layout has no tenant or VM field. + +## WireGuard gateway peer + +```text +fdac : region : node : tenant (32 bits) : padding (32 bits) : peer number +``` + +Example: peer `9` of tenant `42` on `wireguard-002` in region `1` is `fdac:1:2:0:2a::9`. + +The node field comes before the tenant. Then each node owns one `/48`, and a host can send the replies for that node to one VM. The node address is `fdac::::1`. See [WireGuard gateway](wireguard-gateway.md). + +## IPv6 router public address + +```text + : tenant (32 bits) : VM number +``` + +Example: block `2001:db8:1:2:3::/80` maps `fdaa:1:0:2a::7` to `2001:db8:1:2:3:0:2a:7`. + +The public address has no padding. A `/80` block has exactly 48 host bits. A mesh address with padding has no public address. See [IPv6 router](ipv6-router.md). + +## HTTP proxy auto-proxy label + +```text +label = base36((VM number << 32) | tenant) +``` + +The value has 48 bits, so a label has at most 10 characters. See [OpenResty](http-proxy/openresty.md#auto-proxy-traffic). diff --git a/docs/networking/index.md b/docs/networking/index.md index 488b1413..efe3f9d6 100644 --- a/docs/networking/index.md +++ b/docs/networking/index.md @@ -41,10 +41,10 @@ Atlas itself is one more `wg0` peer on each host, outside the mesh peer list. Ho Each VM gets a stable address. The Atlas app derives it when it creates the VM: ```text -fdaa : region : tenant : VM number +fdaa : region : tenant : padding : VM number ``` -For example, `fdaa:1:0:2::3` is VM `3` of tenant `2` in region `1`. +For example, `fdaa:1:0:2::3` is VM `3` of tenant `2` in region `1`. See [address formats](address-formats.md) for every layout. The address stays with the VM when it moves. WG Mesh checks a learned host location for each packet and starts a lookup when it has no valid entry. diff --git a/docs/networking/ipv6-router.md b/docs/networking/ipv6-router.md index b78c4efd..ef9ad7ee 100644 --- a/docs/networking/ipv6-router.md +++ b/docs/networking/ipv6-router.md @@ -24,7 +24,7 @@ private = fdaa | region 16 bits | tenant 32 bits | padding 48 bits | VM 16 bits For block `2001:db8:1:2:3::/80`, mesh address `fdaa:1:0:2a::7` maps to `2001:db8:1:2:3:0:2a:7`. The tenant and the VM number keep their own hextets. -The block must be inside `2000::/3`, with a prefix from `/3` to `/80`. In a shorter block, the bits between the block and the host part are zero. A mesh address with padding, such as the Atlas address, has no public address. +The block must be inside `2000::/3`, with a prefix from `/3` to `/80`. In a shorter block, the bits between the block and the host part are zero. A mesh address with padding, such as the Atlas address, has no public address. See [address formats](address-formats.md). ::: details Try the IPv6 address mapping in Python diff --git a/docs/networking/wg-mesh/index.md b/docs/networking/wg-mesh/index.md index 8135c656..cc874423 100644 --- a/docs/networking/wg-mesh/index.md +++ b/docs/networking/wg-mesh/index.md @@ -6,7 +6,7 @@ This page lists the exact rules and state of WG Mesh. Read [how VMs reach each o | Range | Use | | --- | --- | -| `fdaa::/16` | VM addresses: `fdaa \| region 16 bits \| tenant 32 bits \| VM ID 64 bits`. | +| `fdaa::/16` | VM addresses: `fdaa \| region 16 bits \| tenant 32 bits \| padding 48 bits \| VM 16 bits`. See [address formats](../address-formats.md). | | `fdab::/16` | Host WireGuard addresses. | For example, `fdaa:1:0:2::3` is region `1`, tenant `2`, VM `3`. The data path reads the tenant field for isolation. diff --git a/docs/networking/wireguard-gateway.md b/docs/networking/wireguard-gateway.md index 399164f3..d64326ad 100644 --- a/docs/networking/wireguard-gateway.md +++ b/docs/networking/wireguard-gateway.md @@ -55,7 +55,7 @@ The gateway eBPF filters drop a packet unless the device and the VM belong to th fdac | region 16 | node 16 | tenant 32 | padding 32 | client 16 ``` -Each node owns one `fdac::::/48`. Atlas reserves `fdaa::/16`, `fdab::/16`, and `fdac::/16`. A customer network that uses one of these ranges conflicts with the routes of its devices, so it must use another range. +Each node owns one `fdac::::/48`. See [address formats](address-formats.md#wireguard-gateway-peer). Atlas reserves `fdaa::/16`, `fdab::/16`, and `fdac::/16`. A customer network that uses one of these ranges conflicts with the routes of its devices, so it must use another range. ## Add a node diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 3f6c382e..cdc5a3a0 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -55,7 +55,7 @@ Find a term by topic. **Atlas** names the whole project. **Atlas app** names its | Term | Meaning | | --- | --- | -| Mesh address | The stable `fdaa::/16` IPv6 address that Atlas derives for a VM from region, tenant, and VM number. | +| Mesh address | The stable `fdaa::/16` IPv6 address that Atlas derives for a VM from region, tenant, and VM number. See [address formats](../networking/address-formats.md). | | Direct public IP | A public address that reaches the VM through its host or provider. | | Routed public IPv6 | A `/128` from a router block that reaches the VM through the IPv6 router VM. | | Public address request | The current Atlas request to attach or detach a public address. Its version prevents an old job from completing a newer request. | diff --git a/services/wg-mesh/bpf/mesh.h b/services/wg-mesh/bpf/mesh.h index 2e02969e..fd823e21 100644 --- a/services/wg-mesh/bpf/mesh.h +++ b/services/wg-mesh/bpf/mesh.h @@ -14,7 +14,7 @@ #include #include -/* VM address: fdaa | region 16 | tenant 32 | VM ID 64. */ +/* VM address: fdaa | region 16 | tenant 32 | padding 48 | VM 16. */ #define VM_PREFIX 0xfdaa /* Host WireGuard address. */ #define UNDERLAY_PREFIX 0xfdab