diff --git a/.vitepress/sidebar.mts b/.vitepress/sidebar.mts index 62e2ed1ff..d156fe96b 100644 --- a/.vitepress/sidebar.mts +++ b/.vitepress/sidebar.mts @@ -56,6 +56,7 @@ const handbook: DefaultTheme.SidebarItem[] = [ { text: 'Benchmarks', link: '/docs/networking/wg-mesh/benchmarks' }, ], }, + { text: 'Address formats', link: '/docs/networking/address-formats' }, { text: 'Host networking', link: '/docs/networking/host-networking' }, { text: 'Public IPs', link: '/docs/networking/public-ips' }, { diff --git a/atlas/atlas/core/mesh_address.py b/atlas/atlas/core/mesh_address.py index d6b0af325..cd87f410b 100644 --- a/atlas/atlas/core/mesh_address.py +++ b/atlas/atlas/core/mesh_address.py @@ -14,9 +14,10 @@ MESH_PREFIX = 0xFDAA MESH_NETWORK = ipaddress.IPv6Network((MESH_PREFIX << 112, 16)) -MAXIMUM_VIRTUAL_MACHINE_NUMBER = 0xFFFFFFFFFFFFFFFF -# Atlas takes the last tenant-0 VM number, so no VM can use its address. -ATLAS_VIRTUAL_MACHINE_NUMBER = MAXIMUM_VIRTUAL_MACHINE_NUMBER +# A VM address is fdaa | region 16 | tenant 32 | padding 48 | VM number 16. The padding is zero. +MAXIMUM_VIRTUAL_MACHINE_NUMBER = 0xFFFF +# Atlas fills the padding and VM number, so no VM can use its address. +ATLAS_VIRTUAL_MACHINE_NUMBER = (1 << 64) - 1 WIREGUARD_PREFIX = 0xFDAB # The prefix and the region take the first 32 bits, so the low 96 bits of the UUID are the peer part. WIREGUARD_PEER_MASK = (1 << 96) - 1 @@ -42,10 +43,6 @@ def get_virtual_machine_mesh_address( frappe.throw( _("Virtual Machine number {0} is too large for a mesh address.").format(virtual_machine_number) ) - if virtual_machine.tenant_id == 0 and virtual_machine_number == ATLAS_VIRTUAL_MACHINE_NUMBER: - frappe.throw( - _("Virtual Machine number {0} is the Atlas mesh address.").format(virtual_machine_number) - ) address = ( (MESH_PREFIX << 112) | (region_id << 96) | (virtual_machine.tenant_id << 64) | virtual_machine_number diff --git a/atlas/atlas/core/test_mesh_address.py b/atlas/atlas/core/test_mesh_address.py index a0d7c3807..9eca40be0 100644 --- a/atlas/atlas/core/test_mesh_address.py +++ b/atlas/atlas/core/test_mesh_address.py @@ -17,21 +17,18 @@ def setUp(self) -> None: self.get_settings_patch.start() self.addCleanup(self.get_settings_patch.stop) - def test_a_wide_vm_number_fills_the_64_bit_field(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551615", tenant_id=7) + def test_the_largest_vm_number_leaves_the_padding_zero(self) -> None: + virtual_machine = frappe._dict(name="vm-7-65535", tenant_id=7) - address = get_virtual_machine_mesh_address(virtual_machine) + self.assertEqual(get_virtual_machine_mesh_address(virtual_machine), "fdaa:1:0:7::ffff") - self.assertEqual(address, "fdaa:1:0:7:ffff:ffff:ffff:ffff") + def test_the_vm_number_comes_after_the_tenant_in_the_name(self) -> None: + virtual_machine = frappe._dict(name="vm-42-0007", tenant_id=42) - def test_a_tenant_zero_vm_cannot_take_the_atlas_address(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551615", tenant_id=0) + self.assertEqual(get_virtual_machine_mesh_address(virtual_machine), "fdaa:1:0:2a::7") - with self.assertRaises(frappe.ValidationError): - get_virtual_machine_mesh_address(virtual_machine) - - def test_a_vm_number_above_64_bits_is_refused(self) -> None: - virtual_machine = frappe._dict(name="vm-18446744073709551616", tenant_id=7) + def test_a_vm_number_above_16_bits_is_refused(self) -> None: + virtual_machine = frappe._dict(name="vm-7-65536", tenant_id=7) with self.assertRaises(frappe.ValidationError): get_virtual_machine_mesh_address(virtual_machine) diff --git a/atlas/service/SPEC.md b/atlas/service/SPEC.md index 0b3cac4ca..57dc86a80 100644 --- a/atlas/service/SPEC.md +++ b/atlas/service/SPEC.md @@ -46,7 +46,7 @@ See the [HTTP proxy specification](../../services/http-proxy/SPEC.md). ## IPv6 Router Server -- The pool needs a prefix of `/84` or shorter, no gateway, and no allocations. +- The pool needs a prefix of `/80` or shorter, no gateway, and no allocations. - Installation fails if the eBPF program is not attached. - Archive is refused while tenant allocations use the pool. diff --git a/atlas/service/core/ipv6_router/address.py b/atlas/service/core/ipv6_router/address.py index c6304d9fc..81f255d8e 100644 --- a/atlas/service/core/ipv6_router/address.py +++ b/atlas/service/core/ipv6_router/address.py @@ -10,8 +10,8 @@ from atlas.vm.core.models import IPV6_INTERNET_DESTINATION # Must match bpf/address.h in services/ipv6-router. -TENANT_BITS = 24 -MACHINE_BITS = 20 +TENANT_BITS = 32 +MACHINE_BITS = 16 MAXIMUM_PREFIX_LENGTH = 128 - TENANT_BITS - MACHINE_BITS @@ -31,11 +31,9 @@ def get_routed_ipv6(prefix: str, mesh_address: str) -> str: tenant = (mesh >> 64) & 0xFFFFFFFF machine = mesh & 0xFFFFFFFFFFFFFFFF - if tenant >= 1 << TENANT_BITS or machine >= 1 << MACHINE_BITS: + if machine >= 1 << MACHINE_BITS: frappe.throw( - _("Mesh address {0} has a tenant or VM number that the routed IPv6 layout cannot hold.").format( - mesh_address - ), + _("Mesh address {0} has padding that the routed IPv6 layout cannot hold.").format(mesh_address), exc=AtlasUserError, ) diff --git a/atlas/service/core/ipv6_router/test_address.py b/atlas/service/core/ipv6_router/test_address.py index fbea0971a..8bd3d87dd 100644 --- a/atlas/service/core/ipv6_router/test_address.py +++ b/atlas/service/core/ipv6_router/test_address.py @@ -7,31 +7,24 @@ class TestRoutedIPv6(UnitTestCase): - def test_the_tenant_and_vm_fill_the_low_44_bits(self) -> None: - self.assertEqual( - get_routed_ipv6("2001:db8:1:2:3::/80", "fdaa:1:0:abcd::5"), - "2001:db8:1:2:3:a:bcd0:5", - ) + def test_the_tenant_and_vm_fill_the_low_48_bits(self) -> None: + self.assertEqual(get_routed_ipv6("2001:db8:1:2:3::/80", "fdaa:1:0:2a::7"), "2001:db8:1:2:3:0:2a:7") - def test_a_shorter_prefix_keeps_zero_reserved_bits(self) -> None: - self.assertEqual(get_routed_ipv6("2001:db8:1::/48", "fdaa:1:0:2::3"), "2001:db8:1::20:3") + def test_a_shorter_prefix_keeps_the_bits_above_the_host_part_zero(self) -> None: + self.assertEqual(get_routed_ipv6("2001:db8:1::/48", "fdaa:1:0:2::3"), "2001:db8:1::2:3") def test_the_largest_tenant_and_vm_fit(self) -> None: self.assertEqual( - get_routed_ipv6("2001:db8::/64", "fdaa:1:ff:ffff::f:ffff"), "2001:db8::fff:ffff:ffff" + get_routed_ipv6("2001:db8::/64", "fdaa:1:ffff:ffff::ffff"), "2001:db8::ffff:ffff:ffff" ) - def test_a_tenant_that_needs_more_than_24_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "cannot hold"): - get_routed_ipv6("2001:db8::/64", "fdaa:1:100:0::1") - - def test_a_vm_number_that_needs_more_than_20_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "cannot hold"): - get_routed_ipv6("2001:db8::/64", "fdaa:1:0:1::10:0") + def test_an_address_with_padding_is_refused(self) -> None: + with self.assertRaisesRegex(frappe.ValidationError, "padding"): + get_routed_ipv6("2001:db8::/64", "fdaa:1:0:1::1:0") - def test_a_block_smaller_than_84_bits_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "/84"): - get_routed_ipv6("2001:db8::/96", "fdaa:1:0:1::1") + def test_a_block_smaller_than_80_bits_is_refused(self) -> None: + with self.assertRaisesRegex(frappe.ValidationError, "/80"): + get_routed_ipv6("2001:db8::/84", "fdaa:1:0:1::1") def test_a_noncanonical_block_is_refused(self) -> None: with self.assertRaisesRegex(frappe.ValidationError, "canonical IPv6 prefix"): diff --git a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py index e77885c3c..513c2cd8b 100644 --- a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py +++ b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py @@ -48,7 +48,7 @@ def test_a_pool_with_allocations_is_refused(self) -> None: self.validate(public_ip_pool(), has_allocations=True) def test_a_small_pool_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "/84"): + with self.assertRaisesRegex(frappe.ValidationError, "/80"): self.validate(public_ip_pool(prefix="2001:db8::/96")) diff --git a/atlas/vm/SPEC.md b/atlas/vm/SPEC.md index 0fe3c577c..fdf2f8ab7 100644 --- a/atlas/vm/SPEC.md +++ b/atlas/vm/SPEC.md @@ -21,7 +21,7 @@ This file identifies code owners and rules that a code change must preserve. The ## Rules to preserve -- The VM record name is the Metal VM ID from the `vm-.#######` series. A deleted name is never reused. +- The VM record name is the Metal VM ID from the `vm--.####` series, one counter for each tenant. The number after the last dash is the VM number in the mesh address, from 1 to 65,535 within the tenant. See [address formats](../../docs/networking/address-formats.md). A deleted name is never reused. - `VirtualMachineService.create` commits a draft and any public address request before it calls Metal. An uncertain response keeps the draft. Only confirmed Metal absence removes it. - Metal is the authority for current host state. `Virtual Machine State` is a cache for lists and image checks. A failed Metal read must not appear as a guest state. - Placement checks capacity under a MariaDB named lock with READ COMMITTED isolation. Keep the lock until the draft or migration reservation commits. CPU can be oversubscribed. Memory and disk cannot. diff --git a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py index 3c1537999..64625f152 100644 --- a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py @@ -359,16 +359,16 @@ def test_request_rejects_boolean_tenant_id(self) -> None: class TestVirtualMachineDocument(UnitTestCase): - def test_autoname_assigns_permanent_virtual_machine_id(self) -> None: - virtual_machine = frappe.new_doc("Virtual Machine") - - with patch.object( - virtual_machine_module, "make_autoname", return_value="vm-0000042" - ) as make_autoname: + def test_each_tenant_numbers_its_virtual_machines_from_one(self) -> None: + self.addCleanup(frappe.db.delete, "Series", {"name": ["in", ["vm-4000000001-", "vm-4000000002-"]]}) + names = [] + for tenant_id in (4_000_000_001, 4_000_000_001, 4_000_000_002): + virtual_machine = frappe.new_doc("Virtual Machine") + virtual_machine.tenant_id = tenant_id virtual_machine.autoname() + names.append(virtual_machine.name) - self.assertEqual(virtual_machine.name, "vm-0000042") - make_autoname.assert_called_once_with("vm-.#######", doc=virtual_machine) + self.assertEqual(names, ["vm-4000000001-0001", "vm-4000000001-0002", "vm-4000000002-0001"]) # New records have no Server, so virtual-field reads must skip Metal lookup. def test_new_document_reads_virtual_fields_without_a_server(self) -> None: diff --git a/atlas/vm/doctype/virtual_machine/virtual_machine.py b/atlas/vm/doctype/virtual_machine/virtual_machine.py index caeb6e5d8..d8462d93e 100644 --- a/atlas/vm/doctype/virtual_machine/virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/virtual_machine.py @@ -69,8 +69,8 @@ class VirtualMachine(Document): # end: auto-generated types def autoname(self) -> None: - """Assign a permanent virtual machine ID.""" - self.name = make_autoname("vm-.#######", doc=self) + """Assign a permanent virtual machine ID. Each tenant has its own counter.""" + self.name = make_autoname(f"vm-{self.tenant_id}-.####", doc=self) @request_cache def get_metal_vm_info(self) -> MetalVirtualMachine | None: diff --git a/docs/networking/address-formats.md b/docs/networking/address-formats.md new file mode 100644 index 000000000..574fd247a --- /dev/null +++ b/docs/networking/address-formats.md @@ -0,0 +1,71 @@ +# Address formats + +Atlas derives each private IPv6 address from IDs. No service looks up an address in a table. This page is the reference for each layout. + +## Rules for every layout + +Each field starts on a 16-bit boundary, so you can read the IDs from the address. The first hextet gives the address type, and the second hextet gives the region. + +The tenant ID has 32 bits. A VM number and a WireGuard peer number have 16 bits each and use the last hextet. + +Padding fields are zero. They hold space for later fields. Do not use them. + +| Prefix | Use | +| --- | --- | +| `fdaa::/16` | VM mesh addresses and the Atlas address | +| `fdab::/16` | Host `wg0` addresses | +| `fdac::/16` | WireGuard gateway peers | + +## VM mesh address + +```text +fdaa : region : tenant (32 bits) : padding (48 bits) : VM number +``` + +Example: `vm-42-0007` in region `1` is `fdaa:1:0:2a::7`. + +The VM name is `vm--`. Each tenant has its own counter, so a VM number is unique only within its tenant. Each tenant has the `/64` `fdaa::::/64`. A tenant can have VM numbers from 1 to 65,535. Atlas refuses a larger number. + +## Atlas address + +```text +fdaa : region : 0 : 0 : ffff : ffff : ffff : ffff +``` + +The Atlas address is in tenant 0 and has all padding bits set. No VM can have this address. + +## Host wg0 address + +```text +fdab : region : low 96 bits of the host UUID +``` + +Example: `fdab:1:7f90:766a:8d2c:2f76:8d76:2045`. This layout has no tenant or VM field. + +## WireGuard gateway peer + +```text +fdac : region : node : tenant (32 bits) : padding (32 bits) : peer number +``` + +Example: peer `9` of tenant `42` on `wireguard-002` in region `1` is `fdac:1:2:0:2a::9`. + +The node field comes before the tenant. Then each node owns one `/48`, and a host can send the replies for that node to one VM. The node address is `fdac::::1`. See [WireGuard gateway](wireguard-gateway.md). + +## IPv6 router public address + +```text + : tenant (32 bits) : VM number +``` + +Example: block `2001:db8:1:2:3::/80` maps `fdaa:1:0:2a::7` to `2001:db8:1:2:3:0:2a:7`. + +The public address has no padding. A `/80` block has exactly 48 host bits. A mesh address with padding has no public address. See [IPv6 router](ipv6-router.md). + +## HTTP proxy auto-proxy label + +```text +label = base36((VM number << 32) | tenant) +``` + +The value has 48 bits, so a label has at most 10 characters. See [OpenResty](http-proxy/openresty.md#auto-proxy-traffic). diff --git a/docs/networking/http-proxy/openresty.md b/docs/networking/http-proxy/openresty.md index 8c364b9d6..104e2182c 100644 --- a/docs/networking/http-proxy/openresty.md +++ b/docs/networking/http-proxy/openresty.md @@ -31,7 +31,7 @@ The wildcard certificate covers site HTTPS traffic. An address of `-` returns `5 An auto-proxy name sends HTTP to a VM without a saved site-map entry. The hostname uses a configured prefix, such as `site-` or `*-vm-`, followed by a base-36 label. OpenResty sends the request to the decoded VM mesh address on port `80`. -Atlas puts the 64-bit VM number above the 32-bit tenant ID: `value = (vm << 32) | tenant`. The VM number is the numeric suffix of its Atlas ID, so `VM-00011` uses `11`. Atlas writes the 96-bit value in lowercase base 36. The regional mesh prefix supplies `fdaa:` when OpenResty decodes the label. +Atlas puts the 16-bit VM number above the 32-bit tenant ID: `value = (vm << 32) | tenant`. The VM number is the numeric suffix of its Atlas ID, so `vm-2-0011` uses `11`. Atlas writes the 48-bit value in lowercase base 36, in at most 10 characters. The regional mesh prefix supplies `fdaa:` when OpenResty decodes the label. ::: details Try the base-36 mapping in Python @@ -62,7 +62,7 @@ print(name, mesh) # site-lpc8lqa.example.com fdaa:1:0:2::b ::: -The proxy accepts only configured host prefixes and a canonical lowercase label. The label has at most 19 digits and no leading zero. Values larger than 96 bits and unsupported prefixes do not route. The control daemon reserves matching site keys so a stored route cannot override an auto-proxy name. +The proxy accepts only configured host prefixes and a canonical lowercase label. The label has at most 10 digits and no leading zero. Values larger than 48 bits and unsupported prefixes do not route. The control daemon reserves matching site keys so a stored route cannot override an auto-proxy name. ::: info Static route, live destination The name encodes tenant and VM identity, not a host location. WG Mesh finds the VM's current Metal host after a move. An auto-proxy name alone does not grant access through tenant or guest firewall rules. diff --git a/docs/networking/index.md b/docs/networking/index.md index 488b14138..efe3f9d60 100644 --- a/docs/networking/index.md +++ b/docs/networking/index.md @@ -41,10 +41,10 @@ Atlas itself is one more `wg0` peer on each host, outside the mesh peer list. Ho Each VM gets a stable address. The Atlas app derives it when it creates the VM: ```text -fdaa : region : tenant : VM number +fdaa : region : tenant : padding : VM number ``` -For example, `fdaa:1:0:2::3` is VM `3` of tenant `2` in region `1`. +For example, `fdaa:1:0:2::3` is VM `3` of tenant `2` in region `1`. See [address formats](address-formats.md) for every layout. The address stays with the VM when it moves. WG Mesh checks a learned host location for each packet and starts a lookup when it has no valid entry. diff --git a/docs/networking/ipv6-router.md b/docs/networking/ipv6-router.md index cc84b41c6..ef9ad7ee8 100644 --- a/docs/networking/ipv6-router.md +++ b/docs/networking/ipv6-router.md @@ -18,13 +18,13 @@ The VM can move while its mesh and public addresses stay the same. WG Mesh finds ## How the address maps ```text -public = | reserved, all zero | tenant 24 bits | VM 20 bits -private = fdaa | region 16 bits | tenant 32 bits | VM 64 bits +public = | tenant 32 bits | VM 16 bits +private = fdaa | region 16 bits | tenant 32 bits | padding 48 bits | VM 16 bits ``` -For block `2001:db8:1:2:3::/80`, mesh address `fdaa:1:0:abcd::5` maps to `2001:db8:1:2:3:a:bcd0:5`. +For block `2001:db8:1:2:3::/80`, mesh address `fdaa:1:0:2a::7` maps to `2001:db8:1:2:3:0:2a:7`. The tenant and the VM number keep their own hextets. -The block must be inside `2000::/3`, with a prefix from `/3` to `/84`. A VM whose tenant ID is 2^24 or more, or whose VM number is 2^20 or more, gets no public address. +The block must be inside `2000::/3`, with a prefix from `/3` to `/80`. In a shorter block, the bits between the block and the host part are zero. A mesh address with padding, such as the Atlas address, has no public address. See [address formats](address-formats.md). ::: details Try the IPv6 address mapping in Python @@ -34,21 +34,21 @@ This example derives the public address and recovers the tenant and VM number fr from ipaddress import IPv6Address, IPv6Network block = IPv6Network("2001:db8:1:2:3::/80") -mesh = IPv6Address("fdaa:1:0:abcd::5") +mesh = IPv6Address("fdaa:1:0:2a::7") region = 1 tenant = (int(mesh) >> 64) & 0xFFFFFFFF vm = int(mesh) & 0xFFFFFFFFFFFFFFFF -assert tenant < 1 << 24 and vm < 1 << 20 +assert vm < 1 << 16 -public = IPv6Address(int(block.network_address) | (tenant << 20) | vm) +public = IPv6Address(int(block.network_address) | (tenant << 16) | vm) suffix = int(public) - int(block.network_address) -decoded_tenant, decoded_vm = divmod(suffix, 1 << 20) +decoded_tenant, decoded_vm = divmod(suffix, 1 << 16) decoded_mesh = IPv6Address( (0xFDAA << 112) | (region << 96) | (decoded_tenant << 64) | decoded_vm ) print(public, decoded_mesh) -# 2001:db8:1:2:3:a:bcd0:5 fdaa:1:0:abcd::5 +# 2001:db8:1:2:3:0:2a:7 fdaa:1:0:2a::7 ``` ::: @@ -61,8 +61,8 @@ The [gateway packet example](wg-mesh/gateways.md#how-the-packet-changes) shows t A `tc` eBPF program on the router's `eth0` ingress rewrites the address and fixes the checksum. For an ICMPv6 error, it also rewrites the address inside the quoted packet. It drops: -- A packet to a block address with a reserved bit set. -- A mesh source from another region, or a tenant or VM number that does not fit the public layout. +- A packet to a block address with a set bit between the block and the host part. +- A mesh source from another region, or a mesh source with padding. - A translated packet that is not TCP, UDP, ICMPv6, or a fragment of one of them. ## Block unassigned traffic diff --git a/docs/networking/public-ips.md b/docs/networking/public-ips.md index 836a0baa3..9e5d659e5 100644 --- a/docs/networking/public-ips.md +++ b/docs/networking/public-ips.md @@ -47,7 +47,7 @@ The router receives the full prefix. Each tenant VM gets a derived `/128`, so At Requirements: -- Prefix length `/84` or shorter. AWS `/80` and Scaleway `/64` qualify. +- Prefix length `/80` or shorter. AWS `/80` and Scaleway `/64` qualify. - No allocations in the pool before router creation. - An Active router before routed automatic assignment. diff --git a/docs/networking/wg-mesh/index.md b/docs/networking/wg-mesh/index.md index 8135c6567..cc8744232 100644 --- a/docs/networking/wg-mesh/index.md +++ b/docs/networking/wg-mesh/index.md @@ -6,7 +6,7 @@ This page lists the exact rules and state of WG Mesh. Read [how VMs reach each o | Range | Use | | --- | --- | -| `fdaa::/16` | VM addresses: `fdaa \| region 16 bits \| tenant 32 bits \| VM ID 64 bits`. | +| `fdaa::/16` | VM addresses: `fdaa \| region 16 bits \| tenant 32 bits \| padding 48 bits \| VM 16 bits`. See [address formats](../address-formats.md). | | `fdab::/16` | Host WireGuard addresses. | For example, `fdaa:1:0:2::3` is region `1`, tenant `2`, VM `3`. The data path reads the tenant field for isolation. diff --git a/docs/networking/wireguard-gateway.md b/docs/networking/wireguard-gateway.md index 7aafe2dd1..d64326ad4 100644 --- a/docs/networking/wireguard-gateway.md +++ b/docs/networking/wireguard-gateway.md @@ -26,11 +26,13 @@ The answer holds the WireGuard settings of the device: | Field | Example | Use | | --- | --- | --- | -| `address` | `fdac:1:2:0:2a:0:9:0/128` | Interface address of the device. | +| `address` | `fdac:1:2:0:2a::9/128` | Interface address of the device. | | `allowed_ips` | `["fdaa:1:0:2a::/64"]` | The VM addresses of the tenant. | | `endpoint` | `wireguard-002.par-1.example.com:51820` | The node that serves the device. | | `public_key` | `` | Public key of that node. | +The `tenant_id` is from 1 to 4,294,967,295, and the `client_id` is from 1 to 65,535. + A repeated request with the same key returns the same settings. A new device goes to the node with the fewest devices. `DELETE /v1/peers` with `{"tenant_id": 42, "client_id": 9}` removes a device, and `GET /v1/peers` lists every device with its `node_id`. `PUT /v1/peers` replaces the complete table, for example to restore it from Central's records. Send each device's `node_id` from the list, because the device address carries the node number. A device without `node_id` goes to the node with the fewest devices. @@ -50,10 +52,10 @@ The gateway eBPF filters drop a packet unless the device and the VM belong to th ## Addresses ```text -fdac | region 16 | node 16 | tenant 32 | client 32 | zero 16 +fdac | region 16 | node 16 | tenant 32 | padding 32 | client 16 ``` -Each node owns one `fdac::::/48`. Atlas reserves `fdaa::/16`, `fdab::/16`, and `fdac::/16`. A customer network that uses one of these ranges conflicts with the routes of its devices, so it must use another range. +Each node owns one `fdac::::/48`. See [address formats](address-formats.md#wireguard-gateway-peer). Atlas reserves `fdaa::/16`, `fdab::/16`, and `fdac::/16`. A customer network that uses one of these ranges conflicts with the routes of its devices, so it must use another range. ## Add a node diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 3f6c382ec..cdc5a3a0f 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -55,7 +55,7 @@ Find a term by topic. **Atlas** names the whole project. **Atlas app** names its | Term | Meaning | | --- | --- | -| Mesh address | The stable `fdaa::/16` IPv6 address that Atlas derives for a VM from region, tenant, and VM number. | +| Mesh address | The stable `fdaa::/16` IPv6 address that Atlas derives for a VM from region, tenant, and VM number. See [address formats](../networking/address-formats.md). | | Direct public IP | A public address that reaches the VM through its host or provider. | | Routed public IPv6 | A `/128` from a router block that reaches the VM through the IPv6 router VM. | | Public address request | The current Atlas request to attach or detach a public address. Its version prevents an old job from completing a newer request. | diff --git a/services/http-proxy/nginx/lua/http/auto_proxy.lua b/services/http-proxy/nginx/lua/http/auto_proxy.lua index 3802e03e1..7390c0f34 100644 --- a/services/http-proxy/nginx/lua/http/auto_proxy.lua +++ b/services/http-proxy/nginx/lua/http/auto_proxy.lua @@ -13,7 +13,7 @@ local function digit_value(byte) end local function append_digit(address_parts, digit) - -- Keep the 96-bit value in six 16-bit address parts. + -- Keep the 48-bit value in three 16-bit address parts. local carry = digit for position = #address_parts, 1, -1 do local value = address_parts[position] * BASE + carry @@ -24,7 +24,7 @@ local function append_digit(address_parts, digit) end local function decode_token(token) - local address_parts = { 0, 0, 0, 0, 0, 0 } + local address_parts = { 0, 0, 0 } for position = 1, #token do local digit = digit_value(token:byte(position)) if not digit or not append_digit(address_parts, digit) then @@ -56,17 +56,8 @@ local function is_allowed_prefix(host_prefix, host_prefixes) end local function format_address(address_prefix, address_parts) - -- IPv6 stores tenant parts before VM parts. - return string.format( - "%s:%x:%x:%x:%x:%x:%x", - address_prefix, - address_parts[5], - address_parts[6], - address_parts[1], - address_parts[2], - address_parts[3], - address_parts[4] - ) + -- The label puts the VM number before the tenant, and the address puts it after. + return string.format("%s:%x:%x::%x", address_prefix, address_parts[2], address_parts[3], address_parts[1]) end function auto_proxy.get_virtual_machine_address(subdomain, address_prefix, host_prefixes) diff --git a/services/http-proxy/tests/test_proxy.py b/services/http-proxy/tests/test_proxy.py index 87281cdb8..5f53b567b 100644 --- a/services/http-proxy/tests/test_proxy.py +++ b/services/http-proxy/tests/test_proxy.py @@ -192,10 +192,10 @@ def test_a_map_entry_never_overrides_an_auto_proxy_host(): [ # A padded token is not the label of the VM it decodes to. "site-02avaxbsw", - # 20 digits cannot hold a 96-bit value. - "site-zzzzzzzzzzzzzzzzzzzz", + # 11 digits cannot hold a 48-bit value. + "site-zzzzzzzzzzz", # One above the widest tenant and VM number pair. - "site-7oiylpimjg5u2ca1yps", + "site-2rrvthnxts", # The prefix is not configured. "shop-2avaxbsw", # A bare label carries no prefix. diff --git a/services/ipv6-router/bpf/address.h b/services/ipv6-router/bpf/address.h index cb8619aee..c24be02d6 100644 --- a/services/ipv6-router/bpf/address.h +++ b/services/ipv6-router/bpf/address.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: AGPL-3.0 */ /* The address layouts and the stateless mapping between them. * - * public = prefix | reserved, all zero | tenant 24 bits | virtual machine 20 bits - * private = fdaa | region 16 bits | tenant 32 bits | virtual machine 64 bits + * public = prefix, at most /80 | tenant 32 bits | virtual machine 16 bits + * private = fdaa | region 16 bits | tenant 32 bits | padding 48 bits, zero | virtual machine 16 bits */ #ifndef ATLAS_IPV6_ROUTER_ADDRESS_H #define ATLAS_IPV6_ROUTER_ADDRESS_H @@ -18,9 +18,9 @@ #define MESH_PREFIX 0xfdaaULL #define MESH_REGION ((MESH_PREFIX << 16) | REGION_ID) -#define TENANT_BITS 24 -#define MACHINE_BITS 20 -#define TENANT_LIMIT (1ULL << TENANT_BITS) +#define TENANT_BITS 32 +#define MACHINE_BITS 16 +#define TENANT_MASK ((1ULL << TENANT_BITS) - 1) #define MACHINE_LIMIT (1ULL << MACHINE_BITS) #define HOST_MASK ((1ULL << (TENANT_BITS + MACHINE_BITS)) - 1) @@ -68,7 +68,7 @@ static __always_inline bool is_in_block(const struct address *address) return (address->high & PUBLIC_HIGH_MASK) == PUBLIC_HIGH && (address->low & PUBLIC_LOW_MASK) == PUBLIC_LOW; } -/* A block address maps to a virtual machine only when its reserved bits are zero. */ +/* A block address maps to a virtual machine only when the bits between the block and the host part are zero. */ static __always_inline bool is_mapped_public(const struct address *address) { return address->high == PUBLIC_HIGH && (address->low & ~HOST_MASK) == PUBLIC_LOW; @@ -79,17 +79,17 @@ static __always_inline struct address to_private(const struct address *public) { struct address private; - private.high = (MESH_REGION << 32) | ((public->low >> MACHINE_BITS) & (TENANT_LIMIT - 1)); + private.high = (MESH_REGION << 32) | ((public->low >> MACHINE_BITS) & TENANT_MASK); private.low = public->low & (MACHINE_LIMIT - 1); return private; } -/* Returns false for another region, or a tenant or machine number that the public layout cannot hold. */ +/* Returns false for another region, or an address with padding, such as the Atlas address. */ static __always_inline bool to_public(const struct address *private, struct address *public) { - __u64 tenant = private->high & 0xffffffffULL; + __u64 tenant = private->high & TENANT_MASK; - if ((private->high >> 32) != MESH_REGION || tenant >= TENANT_LIMIT || private->low >= MACHINE_LIMIT) + if ((private->high >> 32) != MESH_REGION || private->low >= MACHINE_LIMIT) return false; public->high = PUBLIC_HIGH; diff --git a/services/ipv6-router/setup.sh b/services/ipv6-router/setup.sh index ba214f28e..92afc7d3b 100755 --- a/services/ipv6-router/setup.sh +++ b/services/ipv6-router/setup.sh @@ -34,7 +34,8 @@ printf '%s\n' "${modules[@]}" > /etc/modules-load.d/atlas-ipv6-router.conf step "write the configuration" install -d -m 0755 "$state_dir" -# The public host part needs 44 bits, so the prefix can be at most /84. + +# The public host part needs 48 bits, so the prefix can be at most /80. read -r mesh_region public_prefix < <(python3 - "$REGION_ID" "$PUBLIC_IPV6_PREFIX" "$state_dir/config.h" <<'PYTHON' import ipaddress import sys @@ -51,8 +52,8 @@ if not 0 <= region_id <= 0xFFFF: sys.exit(f"REGION_ID {region_id} is not a 16-bit number") if not network.subnet_of(ipaddress.IPv6Network("2000::/3")): sys.exit(f"PUBLIC_IPV6_PREFIX {network} is outside 2000::/3") -if network.prefixlen > 84: - sys.exit(f"PUBLIC_IPV6_PREFIX {network} is longer than /84") +if network.prefixlen > 80: + sys.exit(f"PUBLIC_IPV6_PREFIX {network} is longer than /80") words = { "PUBLIC_HIGH": int(network.network_address) >> 64, diff --git a/services/wg-gateway/bpf/gateway.c b/services/wg-gateway/bpf/gateway.c index b4e407550..e9b701df9 100644 --- a/services/wg-gateway/bpf/gateway.c +++ b/services/wg-gateway/bpf/gateway.c @@ -16,8 +16,8 @@ static __always_inline int valid_client(const struct in6_addr *client, const str client->s6_addr16[3] == vm->s6_addr16[2] && client->s6_addr16[4] == vm->s6_addr16[3] && (client->s6_addr16[3] || client->s6_addr16[4]) && - (client->s6_addr16[5] || client->s6_addr16[6]) && - client->s6_addr16[7] == 0; + client->s6_addr16[5] == 0 && client->s6_addr16[6] == 0 && + client->s6_addr16[7]; } SEC("tc/client") diff --git a/services/wg-gateway/daemon/gatewayd/main.py b/services/wg-gateway/daemon/gatewayd/main.py index 17d25f6a7..bb2a95e51 100644 --- a/services/wg-gateway/daemon/gatewayd/main.py +++ b/services/wg-gateway/daemon/gatewayd/main.py @@ -70,7 +70,7 @@ class PeerSettings(BaseModel): tenant_id: int = Field(examples=[42], description="Tenant of the peer.") client_id: int = Field(examples=[9], description="Peer number within the tenant.") - address: str = Field(examples=["fdac:1:2:0:2a:0:9:0/128"], description="Interface address of the peer.") + address: str = Field(examples=["fdac:1:2:0:2a::9/128"], description="Interface address of the peer.") allowed_ips: list[str] = Field( examples=[["fdaa:1:0:2a::/64"]], description="The VM addresses of the tenant. The peer puts them in its own AllowedIPs.", diff --git a/services/wg-gateway/daemon/gatewayd/peers.py b/services/wg-gateway/daemon/gatewayd/peers.py index 11bf0566f..2e63f35c4 100644 --- a/services/wg-gateway/daemon/gatewayd/peers.py +++ b/services/wg-gateway/daemon/gatewayd/peers.py @@ -14,7 +14,8 @@ PEERS_KIND = "peers" CLIENT_PREFIX = 0xFDAC MESH_PREFIX = 0xFDAA -FIELD_LIMIT = (1 << 32) - 1 +TENANT_LIMIT = (1 << 32) - 1 +CLIENT_LIMIT = (1 << 16) - 1 PUBLIC_KEY_PATTERN = re.compile(r"[A-Za-z0-9+/]{43}=") NODE_ID_PATTERN = re.compile(r"wireguard-[0-9]{3,5}") INTERFACE = "wg0" @@ -28,9 +29,12 @@ def get_identity(tenant_id: int, client_id: int) -> str: def validate_device(tenant_id: int, client_id: int, public_key: str) -> None: """Reject a device that the address layout or WireGuard cannot hold.""" - for label, value in (("tenant_id", tenant_id), ("client_id", client_id)): - if not 1 <= value <= FIELD_LIMIT: - raise HTTPException(status_code=400, detail=f"{label} must be from 1 to {FIELD_LIMIT}") + for label, value, limit in ( + ("tenant_id", tenant_id, TENANT_LIMIT), + ("client_id", client_id, CLIENT_LIMIT), + ): + if not 1 <= value <= limit: + raise HTTPException(status_code=400, detail=f"{label} must be from 1 to {limit}") if not PUBLIC_KEY_PATTERN.fullmatch(public_key): raise HTTPException(status_code=400, detail="public_key must be a 44-character WireGuard key") @@ -179,13 +183,7 @@ def get_credentials(self, identity: str, peer: dict[str, str]) -> dict[str, obje # The node number is part of the record name, so an archived node still gives the address. gateway_id = int(peer["node_id"].rsplit("-", 1)[-1]) region = self.config.region_id - address = ( - (CLIENT_PREFIX << 112) - | (region << 96) - | (gateway_id << 80) - | (tenant_id << 48) - | (client_id << 16) - ) + address = (CLIENT_PREFIX << 112) | (region << 96) | (gateway_id << 80) | (tenant_id << 48) | client_id tenant_network = (MESH_PREFIX << 112) | (region << 96) | (tenant_id << 64) return { "tenant_id": tenant_id, diff --git a/services/wg-gateway/daemon/tests/test_peers.py b/services/wg-gateway/daemon/tests/test_peers.py index f83cfaf73..5aecfc7d9 100644 --- a/services/wg-gateway/daemon/tests/test_peers.py +++ b/services/wg-gateway/daemon/tests/test_peers.py @@ -7,7 +7,7 @@ from fastapi import HTTPException from gatewayd.config import ConfigError, GatewayConfig, Node, load -from gatewayd.peers import PEERS_KIND, PeerState +from gatewayd.peers import PEERS_KIND, PeerState, validate_device KEY_A = "A" * 43 + "=" KEY_B = "B" * 43 + "=" @@ -50,7 +50,7 @@ def test_credentials_carry_the_node_tenant_and_client_in_the_address(tmp_path): credentials = register(peers, {}, "42:9", KEY_A, "wireguard-007") - assert credentials["address"] == "fdac:2:7:0:2a:0:9:0/128" + assert credentials["address"] == "fdac:2:7:0:2a::9/128" assert credentials["allowed_ips"] == ["fdaa:2:0:2a::/64"] assert credentials["endpoint"] == "wireguard-007.par-1.example.com:51821" assert credentials["public_key"] == "node-7-public" @@ -103,7 +103,7 @@ def test_only_this_node_applies_its_devices(tmp_path): assert peers.applied.count("[Peer]") == 1 assert f"PublicKey = {KEY_A}" in peers.applied - assert "AllowedIPs = fdac:2:1:0:1:0:1:0/128" in peers.applied + assert "AllowedIPs = fdac:2:1:0:1::1/128" in peers.applied def test_a_repeated_registration_keeps_the_first_node(tmp_path): @@ -162,7 +162,7 @@ def test_a_device_of_an_archived_node_is_kept_without_an_endpoint(tmp_path): asyncio.run(peers.apply(state, Mutation(kind=PEERS_KIND, action="replace", value=table))) credentials = peers.get_credentials("1:1", state[PEERS_KIND]["1:1"]) - assert credentials["address"] == "fdac:2:2:0:1:0:1:0/128" + assert credentials["address"] == "fdac:2:2:0:1::1/128" assert (credentials["endpoint"], credentials["public_key"]) == ("", "") with pytest.raises(HTTPException): register(peers, state, "1:2", KEY_B, "wireguard-002") @@ -224,3 +224,9 @@ def apply_wireguard(self, state): with pytest.raises(HTTPException): register(peers, state, "1:1", KEY_A, "wireguard-001") assert state == {PEERS_KIND: {}} + + +def test_a_client_number_must_fit_one_hextet(): + validate_device(1, 65535, KEY_A) + with pytest.raises(HTTPException): + validate_device(1, 65536, KEY_A) diff --git a/services/wg-mesh/bpf/mesh.h b/services/wg-mesh/bpf/mesh.h index 2e02969e7..fd823e217 100644 --- a/services/wg-mesh/bpf/mesh.h +++ b/services/wg-mesh/bpf/mesh.h @@ -14,7 +14,7 @@ #include #include -/* VM address: fdaa | region 16 | tenant 32 | VM ID 64. */ +/* VM address: fdaa | region 16 | tenant 32 | padding 48 | VM 16. */ #define VM_PREFIX 0xfdaa /* Host WireGuard address. */ #define UNDERLAY_PREFIX 0xfdab