diff --git a/.vitepress/sidebar.mts b/.vitepress/sidebar.mts index bc98c86ec..ab0012080 100644 --- a/.vitepress/sidebar.mts +++ b/.vitepress/sidebar.mts @@ -19,6 +19,7 @@ const handbook: DefaultTheme.SidebarItem[] = [ { text: 'Regional settings', link: '/docs/region/configuration' }, { text: 'Provision a host', link: '/docs/region/' }, { text: 'Metal daemon', link: '/docs/region/metald' }, + { text: 'Atlas access to hosts', link: '/docs/region/host-access' }, { text: 'Host sync', link: '/docs/region/host-sync' }, { text: 'Add a provider', link: '/docs/region/provider-guide' }, ], diff --git a/atlas/atlas/SPEC.md b/atlas/atlas/SPEC.md index 37321121f..4107f964e 100644 --- a/atlas/atlas/SPEC.md +++ b/atlas/atlas/SPEC.md @@ -46,7 +46,7 @@ Domain code reaches a provider only through `ServerProvider`. Implementations ar ## SSH tasks -- `target` is a Dynamic Link. The task connects to the target's `ssh_host`. +- `target` is a Dynamic Link. The task connects to the target's `ssh_host` through its `get_ssh_proxy_command()`. A VM uses its host as the proxy. - A task stores no credentials. It uses the Atlas host identity. - `script` and `environment` are plain text. Use `SSHRunner` directly for secret data. diff --git a/atlas/atlas/core/artifacts.py b/atlas/atlas/core/artifacts.py index 1d59e9aa5..01adf47a7 100644 --- a/atlas/atlas/core/artifacts.py +++ b/atlas/atlas/core/artifacts.py @@ -82,11 +82,21 @@ def get_linked_files() -> set[str]: def get_download_url(file_name: str) -> str: - """Return the URL a host uses to download one published File. + """Return the URL a Metal host or a tenant-0 VM uses to download one published File.""" + return get_file_url(file_name, get_internal_base_url()) - `atlas_base_url` in the site configuration names an address that a host can - reach, which the site's own URL is not during local development. - """ + +def get_internal_base_url() -> str: + """Return `atlas_internal_url`, which hosts and tenant-0 VMs reach, or the public URL.""" + return frappe.conf.atlas_internal_url or get_public_base_url() + + +def get_public_base_url() -> str: + """Return `atlas_base_url`, which names a public address when the site URL is not one.""" + return frappe.conf.atlas_base_url or frappe.utils.get_url(allow_header_override=False) + + +def get_file_url(file_name: str, base_url: str) -> str: + """Return the URL of one File below a base URL.""" file_url = frappe.db.get_value("File", file_name, "file_url") - base_url = frappe.conf.atlas_base_url or frappe.utils.get_url(allow_header_override=False) return f"{base_url.rstrip('/')}{file_url}" diff --git a/atlas/atlas/core/mesh_address.py b/atlas/atlas/core/mesh_address.py index 433567c43..d6b0af325 100644 --- a/atlas/atlas/core/mesh_address.py +++ b/atlas/atlas/core/mesh_address.py @@ -2,6 +2,7 @@ import ipaddress from typing import TYPE_CHECKING, cast +from uuid import UUID import frappe from frappe import _ @@ -14,21 +15,37 @@ MESH_PREFIX = 0xFDAA MESH_NETWORK = ipaddress.IPv6Network((MESH_PREFIX << 112, 16)) MAXIMUM_VIRTUAL_MACHINE_NUMBER = 0xFFFFFFFFFFFFFFFF +# Atlas takes the last tenant-0 VM number, so no VM can use its address. +ATLAS_VIRTUAL_MACHINE_NUMBER = MAXIMUM_VIRTUAL_MACHINE_NUMBER +WIREGUARD_PREFIX = 0xFDAB +# The prefix and the region take the first 32 bits, so the low 96 bits of the UUID are the peer part. +WIREGUARD_PEER_MASK = (1 << 96) - 1 -def get_virtual_machine_mesh_address(virtual_machine: Document | frappe._dict) -> str: - """Return the mesh address from stable Atlas request metadata.""" - settings = cast("AtlasSettings", frappe.get_single("Atlas Settings")) - region_id = settings.region_id +def validate_region_id(region_id: int) -> None: + """Refuse a region ID that does not fit in one IPv6 field.""" if not 0 <= region_id <= 0xFFFF: frappe.throw(_("Atlas Settings region ID must be a 16-bit unsigned integer.")) + +def get_virtual_machine_mesh_address( + virtual_machine: Document | frappe._dict, region_id: int | None = None +) -> str: + """Return the mesh address from stable Atlas request metadata. A caller in a loop passes the region.""" + if region_id is None: + region_id = cast("AtlasSettings", frappe.get_single("Atlas Settings")).region_id + validate_region_id(region_id) + virtual_machine_name = cast(str, virtual_machine.name) virtual_machine_number = int(virtual_machine_name.rsplit("-", 1)[-1]) if virtual_machine_number > MAXIMUM_VIRTUAL_MACHINE_NUMBER: frappe.throw( _("Virtual Machine number {0} is too large for a mesh address.").format(virtual_machine_number) ) + if virtual_machine.tenant_id == 0 and virtual_machine_number == ATLAS_VIRTUAL_MACHINE_NUMBER: + frappe.throw( + _("Virtual Machine number {0} is the Atlas mesh address.").format(virtual_machine_number) + ) address = ( (MESH_PREFIX << 112) | (region_id << 96) | (virtual_machine.tenant_id << 64) | virtual_machine_number @@ -36,9 +53,23 @@ def get_virtual_machine_mesh_address(virtual_machine: Document | frappe._dict) - return str(ipaddress.IPv6Address(address)) +def get_atlas_mesh_address(region_id: int) -> str: + """Return the tenant-0 mesh address of Atlas in one region.""" + validate_region_id(region_id) + + return str(ipaddress.IPv6Address((MESH_PREFIX << 112) | (region_id << 96) | ATLAS_VIRTUAL_MACHINE_NUMBER)) + + def get_region_mesh_address_prefix(region_id: int) -> str: """Return the leading hextets for VM mesh addresses in one region.""" - if not 0 <= region_id <= 0xFFFF: - frappe.throw(_("Atlas Settings region ID must be a 16-bit unsigned integer.")) + validate_region_id(region_id) return f"{MESH_PREFIX:x}:{region_id:x}" + + +def get_wireguard_ip_address(peer_id: UUID, region_id: int) -> str: + """Return the fdab::/16 wg0 address of one host or Atlas peer.""" + validate_region_id(region_id) + + address = (WIREGUARD_PREFIX << 112) | (region_id << 96) | (peer_id.int & WIREGUARD_PEER_MASK) + return str(ipaddress.IPv6Address(address)) diff --git a/atlas/atlas/core/server_providers/aws/provider.py b/atlas/atlas/core/server_providers/aws/provider.py index 7cab5e11d..ccfe352e9 100644 --- a/atlas/atlas/core/server_providers/aws/provider.py +++ b/atlas/atlas/core/server_providers/aws/provider.py @@ -117,6 +117,38 @@ def ensure_server(self, request: ServerCreateRequest) -> ProviderServer: """Return the named AWS instance, and create it when necessary.""" return self.servers.ensure(request) + @override + def import_server(self, server: "MetalServer") -> None: + """Match an existing instance to the catalog. Its storage volume must exist already.""" + instance = self.servers.fetch(server.provider_server_id) + if not instance.get("ImageId"): + raise AwsError(f"AWS instance {server.provider_server_id} has no image") + # Public addresses attach to the primary interface, so it must be in the Atlas subnet. + if instance.get("SubnetId") != self.configuration.subnet_id: + raise AwsError( + f"AWS instance {server.provider_server_id} is in subnet {instance.get('SubnetId')}, " + f"not the Atlas subnet {self.configuration.subnet_id}" + ) + instance_type = instance.get("InstanceType") + if not isinstance(instance_type, str) or not frappe.db.exists("Metal Server Size", instance_type): + raise AwsError( + f"No Metal Server Size matches {instance_type}. Sync the Metal Server Size catalog first." + ) + server.server_size = instance_type + # The catalog keeps only the newest image of each version, so match the version. + images = self.client.call("ec2", "describe_images", ImageIds=[instance["ImageId"]]).get("Images", []) + versions = self.catalog.get_server_images(images) + if not versions or not frappe.db.exists("Metal Server Image", versions[0].name): + raise AwsError( + f"No Metal Server Image matches image {instance['ImageId']}. Sync the Metal Server Image catalog first." + ) + server.server_image = versions[0].name + tags = { + tag.get("Key"): tag.get("Value") for tag in instance.get("Tags") or [] if isinstance(tag, Mapping) + } + server.title = tags.get("Name") or server.provider_server_id + self.apply_provider_server(server, self.servers.to_provider_server(instance)) + @override def prepare_server(self, server: "MetalServer") -> None: """Prepare the AWS instance before Secure Shell access.""" @@ -143,16 +175,6 @@ def configure_server_network(self, server: "MetalServer") -> None: ) self.wait_for_private_address(server) - @override - def metald_listen_address(self, server: "MetalServer") -> str: - """Return the primary interface address behind the internet gateway.""" - metadata = frappe.parse_json(server.provider_metadata or "{}") - instance = metadata.get("instance") if isinstance(metadata, Mapping) else None - address = instance.get("PrivateIpAddress") if isinstance(instance, Mapping) else None - if not isinstance(address, str) or not address: - raise AwsError("Atlas server has no AWS primary private IPv4 address") - return address - @override def storage_pool_device(self, server: "MetalServer") -> str: """Return the stable device path of the EBS volume for the storage pool.""" @@ -303,9 +325,7 @@ def uplink_interface(self, server: "MetalServer") -> str: """Return the guest device name that carries the AWS default route.""" from atlas.atlas.core.ssh import SSHRunner - result = SSHRunner(server.public_ipv4_address).run_command( - "ip -4 -o route show default", timeout_seconds=15 - ) + result = SSHRunner(server.ssh_host).run_command("ip -4 -o route show default", timeout_seconds=15) fields = result.output.split() if result.exit_code != 0 or "dev" not in fields: raise AwsError(f"Atlas server {server.name} has no default route device") diff --git a/atlas/atlas/core/server_providers/aws/test_provider.py b/atlas/atlas/core/server_providers/aws/test_provider.py index b22fdb6ad..f42334ddc 100644 --- a/atlas/atlas/core/server_providers/aws/test_provider.py +++ b/atlas/atlas/core/server_providers/aws/test_provider.py @@ -247,18 +247,6 @@ def test_configure_server_network_fails_without_a_mesh_mac_address(self) -> None with self.assertRaises(AwsError): provider.configure_server_network(server) - def test_metald_binds_the_primary_interface_not_the_mesh_interface(self) -> None: - provider = self.provider() - server = self.server() - server.provider_metadata = json.dumps( - { - "instance": {"PrivateIpAddress": "10.1.8.189", "PublicIpAddress": "56.155.92.65"}, - "mesh_interface": {"PrivateIpAddress": "10.1.0.240"}, - } - ) - - self.assertEqual(provider.metald_listen_address(server), "10.1.8.189") - def test_a_public_address_attaches_to_the_primary_interface(self) -> None: provider = self.provider() provider.ip_addresses = Mock() @@ -294,14 +282,6 @@ def test_a_public_address_needs_the_primary_interface(self) -> None: with self.assertRaisesRegex(AwsError, "primary network interface"): provider.attach_public_ip_address("eipalloc-1", "203.0.113.9", server) - def test_metald_needs_the_primary_private_address(self) -> None: - provider = self.provider() - server = self.server() - server.provider_metadata = json.dumps({"instance": {"PublicIpAddress": "56.155.92.65"}}) - - with self.assertRaisesRegex(AwsError, "primary private IPv4 address"): - provider.metald_listen_address(server) - def test_the_storage_pool_device_names_the_attached_volume(self) -> None: provider = self.provider() server = self.server() @@ -376,6 +356,42 @@ def test_uplink_interface_fails_without_a_default_route(self) -> None: with patch("atlas.atlas.core.ssh.SSHRunner", return_value=runner), self.assertRaises(AwsError): provider.uplink_interface(self.server()) + def test_an_import_outside_the_atlas_subnet_is_refused(self) -> None: + """Public addresses attach to the primary interface, so it must be in the Atlas subnet.""" + provider = self.provider() + provider.configuration = SimpleNamespace(subnet_id="subnet-atlas") + provider.servers.fetch.return_value = {"ImageId": "ami-1", "SubnetId": "subnet-other"} + + with self.assertRaisesRegex(AwsError, "not the Atlas subnet subnet-atlas"): + provider.import_server(self.server("i-1")) + + def test_an_import_matches_an_older_image_of_a_catalog_version(self) -> None: + """The catalog keeps only the newest image, and an instance can run an older one.""" + provider = self.provider() + provider.configuration = SimpleNamespace(subnet_id="subnet-atlas") + provider.servers.fetch.return_value = { + "ImageId": "ami-old", + "SubnetId": "subnet-atlas", + "InstanceType": "c7i.xlarge", + "Tags": [{"Key": "Name", "Value": "osa-host"}], + } + provider.client.call.return_value = { + "Images": [ + { + "ImageId": "ami-old", + "Name": "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20260904", + "CreationDate": "2026-09-04T11:45:55.000Z", + } + ] + } + provider.apply_provider_server = Mock() + server = self.server("i-1") + + with patch("atlas.atlas.core.server_providers.aws.provider.frappe.db.exists", return_value=True): + provider.import_server(server) + + self.assertEqual((server.server_image, server.title), ("Ubuntu_24.04", "osa-host")) + def provider(self) -> AwsProvider: provider = object.__new__(AwsProvider) provider.settings = SimpleNamespace( @@ -401,6 +417,7 @@ def server(provider_server_id: str | None = None) -> SimpleNamespace: provider_server_id=provider_server_id, provider_metadata="{}", public_ipv4_address="203.0.113.1", + ssh_host="203.0.113.1", private_ipv4_address=None, public_network_interface=None, private_network_interface=None, diff --git a/atlas/atlas/core/server_providers/aws/test_volumes.py b/atlas/atlas/core/server_providers/aws/test_volumes.py index 996c178fb..e596f2534 100644 --- a/atlas/atlas/core/server_providers/aws/test_volumes.py +++ b/atlas/atlas/core/server_providers/aws/test_volumes.py @@ -73,7 +73,7 @@ def test_grow_runs_the_script_for_the_storage_pool(self) -> None: volumes = self.volumes( modifications=[{"StartTime": datetime.now(UTC), "ModificationState": "optimizing"}] ) - volumes.provider.storage_pool_device.return_value = "/dev/disk/by-id/pool" + volumes.provider.get_storage_pool_device.return_value = "/dev/disk/by-id/pool" volumes.provider.poll.side_effect = lambda operation, **_: operation() server = self.server() task = SimpleNamespace(name="SSH-1", result=SimpleNamespace(is_success=True)) diff --git a/atlas/atlas/core/server_providers/aws/volumes.py b/atlas/atlas/core/server_providers/aws/volumes.py index 76e108cd7..6e2e3f709 100644 --- a/atlas/atlas/core/server_providers/aws/volumes.py +++ b/atlas/atlas/core/server_providers/aws/volumes.py @@ -85,7 +85,7 @@ def grow(self, server: "MetalServer", kind: str) -> None: ) environment = {"TARGET": kind} if kind == "storage": - environment["STORAGE_POOL_DEVICE"] = self.provider.storage_pool_device(server) + environment["STORAGE_POOL_DEVICE"] = self.provider.get_storage_pool_device(server) task = SSHTask.create_for_script_file( target_type=server.doctype, target=server.name, diff --git a/atlas/atlas/core/server_providers/base.py b/atlas/atlas/core/server_providers/base.py index 9ec75ba58..1c716a1e5 100644 --- a/atlas/atlas/core/server_providers/base.py +++ b/atlas/atlas/core/server_providers/base.py @@ -171,22 +171,17 @@ def configure_server_network(self, server: "MetalServer") -> None: """Configure the provider network after Secure Shell access is ready.""" ... - def metald_listen_address(self, server: "MetalServer") -> str: - """Return the configured metald address.""" - address = ( - server.public_ipv4_address - if self.settings.use_public_ip_for_metald - else server.private_ipv4_address - ) - if not address: - raise self.error_class(f"Atlas server {server.name} has no address for metald") - return address - @abstractmethod def storage_pool_device(self, server: "MetalServer") -> str: """Return the raw block device for the virtual machine storage pool.""" ... + def get_storage_pool_device(self, server: "MetalServer") -> str: + """Return the device that registration or import stored, else the provider device.""" + metadata = frappe.parse_json(server.provider_metadata or "{}") + device = metadata.get("storage_pool_device") if isinstance(metadata, Mapping) else None + return device or self.storage_pool_device(server) + @abstractmethod def set_power_state(self, provider_server_id: str, action: ServerPowerAction) -> None: """Apply one power action to a provider server.""" @@ -197,6 +192,18 @@ def delete_server(self, provider_server_id: str, provider_metadata: Mapping[str, """Delete a provider server and its owned resources if they exist.""" ... + def import_server(self, server: "MetalServer") -> None: + """Fill a Metal Server from a provider server that Atlas did not create.""" + raise UnsupportedProviderOperation("server import") + + def find_catalog_record(self, doctype: str, matches: Callable[[Mapping], bool], label: str) -> str: + """Return the catalog record whose provider metadata matches an imported server.""" + for name, metadata in frappe.get_all(doctype, fields=["name", "provider_metadata"], as_list=True): + values = frappe.parse_json(metadata or "{}") + if isinstance(values, Mapping) and matches(values): + return name + raise self.error_class(f"No {doctype} matches {label}. Sync the {doctype} catalog first.") + def reserve_public_ip_address(self, version: int) -> ReservedIPAddress: """Reserve one public IPv4 address or IPv6 block.""" raise UnsupportedProviderOperation(f"public IPv{version} address reservation") @@ -233,7 +240,7 @@ def wait_for_private_address(self, server: "MetalServer") -> None: def get_private_network_mac_address() -> str | None: """Return the interface MAC once the server has its private network address.""" try: - result = SSHRunner(server.public_ipv4_address).run_command( + result = SSHRunner(server.ssh_host).run_command( f"ip -4 -o addr show dev {device} scope global && cat /sys/class/net/{device}/address", timeout_seconds=15, ) diff --git a/atlas/atlas/core/server_providers/generic/provider.py b/atlas/atlas/core/server_providers/generic/provider.py index 6a37a27b2..cecfdb350 100644 --- a/atlas/atlas/core/server_providers/generic/provider.py +++ b/atlas/atlas/core/server_providers/generic/provider.py @@ -47,7 +47,7 @@ def validate_settings(self) -> None: @override def validate_server(self, server: "MetalServer") -> None: """Require the storage pool device that host registration stores.""" - self.storage_pool_device(server) + self.get_storage_pool_device(server) @override def validate_credentials(self) -> bool: @@ -86,12 +86,8 @@ def configure_server_network(self, server: "MetalServer") -> None: @override def storage_pool_device(self, server: "MetalServer") -> str: - """Return the storage pool device that host registration chose.""" - metadata = frappe.parse_json(server.provider_metadata or "{}") - device = metadata.get("storage_pool_device") if isinstance(metadata, dict) else None - if not device: - raise GenericError(f"Metal Server {server.name} has no registered storage pool device") - return device + """Refuse a host without the storage pool device that registration stores.""" + raise GenericError(f"Metal Server {server.name} has no registered storage pool device") @override def set_power_state(self, provider_server_id: str, action: ServerPowerAction) -> None: diff --git a/atlas/atlas/core/server_providers/generic/test_provider.py b/atlas/atlas/core/server_providers/generic/test_provider.py index 505285ea1..2e2a1c38a 100644 --- a/atlas/atlas/core/server_providers/generic/test_provider.py +++ b/atlas/atlas/core/server_providers/generic/test_provider.py @@ -44,7 +44,9 @@ def test_network_setup_only_checks_the_private_address(self) -> None: run_setup_script.assert_not_called() def test_storage_pool_device_is_the_registered_device(self) -> None: - self.assertEqual(GenericProvider(self.settings()).storage_pool_device(self.server()), "/dev/nvme1n1") + self.assertEqual( + GenericProvider(self.settings()).get_storage_pool_device(self.server()), "/dev/nvme1n1" + ) def test_public_address_attaches_as_itself(self) -> None: provider = GenericProvider(self.settings()) diff --git a/atlas/atlas/core/server_providers/scaleway/provider.py b/atlas/atlas/core/server_providers/scaleway/provider.py index 65d2f334f..4d4bef7c6 100644 --- a/atlas/atlas/core/server_providers/scaleway/provider.py +++ b/atlas/atlas/core/server_providers/scaleway/provider.py @@ -114,6 +114,45 @@ def ensure_server(self, request: ServerCreateRequest) -> ProviderServer: """Return the Scaleway server for the identity key.""" return self.servers.ensure(request) + @override + def import_server(self, server: "MetalServer") -> None: + """Match an existing Elastic Metal server to the catalog and give it the private network option.""" + remote_server = self.servers.fetch(server.provider_server_id) + operating_system_id = (remote_server.get("install") or {}).get("os_id") + if not remote_server.get("offer_id") or not operating_system_id: + raise ScalewayError( + f"Scaleway server {server.provider_server_id} has no offer or operating system" + ) + server.server_size, subscription_period = self.find_offer(remote_server.get("offer_id")) + server.server_image = self.find_catalog_record( + "Metal Server Image", + lambda metadata: metadata.get("id") == operating_system_id, + f"operating system {operating_system_id}", + ) + server.title = remote_server.get("name") or server.provider_server_id + size = frappe.get_doc("Metal Server Size", server.server_size) + self.servers.ensure_private_network_option( + remote_server, self.catalog.get_private_network_option_id(size, subscription_period) + ) + self.apply_provider_server(server, self.servers.to_provider_server(remote_server)) + + def find_offer(self, offer_id: object) -> tuple[str, str]: + """Return the Metal Server Size and the subscription period of one Scaleway offer.""" + for name, metadata in frappe.get_all( + "Metal Server Size", fields=["name", "provider_metadata"], as_list=True + ): + values = frappe.parse_json(metadata or "{}") + for period, offer in values.items() if isinstance(values, Mapping) else (): + if not isinstance(offer, Mapping): + continue + if offer_id == offer.get("monthly_offer_id"): + return name, "monthly" + if offer_id == offer.get("id"): + return name, period + raise ScalewayError( + f"No Metal Server Size matches offer {offer_id}. Sync the Metal Server Size catalog first." + ) + @override def prepare_server(self, server: "MetalServer") -> None: """Prepare the Scaleway server before Secure Shell access.""" diff --git a/atlas/atlas/core/server_providers/scaleway/servers.py b/atlas/atlas/core/server_providers/scaleway/servers.py index 66dd263cc..1106b1af7 100644 --- a/atlas/atlas/core/server_providers/scaleway/servers.py +++ b/atlas/atlas/core/server_providers/scaleway/servers.py @@ -95,6 +95,17 @@ def fetch(self, provider_server_id: str) -> Mapping: "GET", f"/baremetal/v1/zones/{self.configuration.zone}/servers/{provider_server_id}" ) + def ensure_private_network_option(self, remote_server: Mapping, option_id: str) -> None: + """Add the private network option that a server created outside Atlas can lack.""" + if any(option.get("id") == option_id for option in remote_server.get("options") or []): + return + + self.client.request( + "POST", + f"/baremetal/v1/zones/{self.configuration.zone}/servers/{remote_server['id']}/options/{option_id}", + json={}, + ) + def ensure_private_network(self, provider_server_id: str) -> Mapping: """Return the private network attachment, and create it when it does not exist.""" private_network = self.private_network(provider_server_id) diff --git a/atlas/atlas/core/server_providers/scaleway/test_provider.py b/atlas/atlas/core/server_providers/scaleway/test_provider.py index 16389bccd..802e7b0c9 100644 --- a/atlas/atlas/core/server_providers/scaleway/test_provider.py +++ b/atlas/atlas/core/server_providers/scaleway/test_provider.py @@ -96,7 +96,7 @@ def test_wait_for_private_address_stores_the_interface_mac(self) -> None: provider = self.provider() server = self.server() server.name = "server-1" - server.public_ipv4_address = "203.0.113.1" + server.ssh_host = "203.0.113.1" server.private_network_interface = "eno1.123" server.private_ipv4_address = "10.1.0.2" server.private_network_mac_address = None @@ -144,6 +144,63 @@ def test_poll_raises_a_permanent_error(self) -> None: operation.assert_called_once_with() + def test_an_imported_server_takes_its_catalog_entries_and_private_network_option(self) -> None: + provider = self.provider() + provider.catalog = Mock() + provider.catalog.get_private_network_option_id.return_value = "monthly-option" + provider.servers.to_provider_server = ScalewayServers.to_provider_server + remote_server = { + "id": "server-1", + "name": "atlas-vm-host", + "offer_id": "monthly-offer", + "install": {"os_id": "ubuntu-id"}, + "ips": [{"version": "IPv4", "address": "203.0.113.4"}], + } + provider.servers.fetch.return_value = remote_server + catalog = { + "Metal Server Size": [ + [ + "EM-A116X-SSD", + json.dumps({"hourly": {"id": "hourly-offer", "monthly_offer_id": "monthly-offer"}}), + ] + ], + "Metal Server Image": [["Ubuntu_24.04", json.dumps({"id": "ubuntu-id"})]], + } + server = self.server("server-1") + + with ( + patch( + "atlas.atlas.core.server_providers.base.frappe.get_all", + side_effect=lambda doctype, **_: catalog[doctype], + ), + patch("atlas.atlas.core.server_providers.scaleway.provider.frappe.get_doc", return_value="size"), + ): + provider.import_server(server) + + self.assertEqual( + (server.server_size, server.server_image, server.title), + ("EM-A116X-SSD", "Ubuntu_24.04", "atlas-vm-host"), + ) + self.assertEqual(server.public_ipv4_address, "203.0.113.4") + provider.catalog.get_private_network_option_id.assert_called_once_with("size", "monthly") + provider.servers.ensure_private_network_option.assert_called_once_with( + remote_server, "monthly-option" + ) + + def test_an_import_without_a_catalog_offer_is_refused(self) -> None: + provider = self.provider() + provider.servers.fetch.return_value = { + "id": "server-1", + "offer_id": "unknown", + "install": {"os_id": "ubuntu-id"}, + } + + with ( + patch("atlas.atlas.core.server_providers.base.frappe.get_all", return_value=[]), + self.assertRaisesRegex(ScalewayError, "Sync the Metal Server Size catalog"), + ): + provider.import_server(self.server("server-1")) + def provider(self) -> ScalewayProvider: provider = object.__new__(ScalewayProvider) provider.settings = SimpleNamespace( @@ -169,6 +226,17 @@ def server(provider_server_id: str | None = None) -> SimpleNamespace: class TestScalewayServers(UnitTestCase): + def test_the_private_network_option_is_added_only_when_missing(self) -> None: + servers = self.servers() + + servers.ensure_private_network_option({"id": "server-1", "options": [{"id": "option"}]}, "option") + servers.client.request.assert_not_called() + + servers.ensure_private_network_option({"id": "server-1", "options": []}, "option") + servers.client.request.assert_called_once_with( + "POST", "/baremetal/v1/zones/fr-par-1/servers/server-1/options/option", json={} + ) + def test_create_uses_catalog_values_and_stable_identity(self) -> None: servers = self.servers() servers.catalog.offer.side_effect = [ diff --git a/atlas/atlas/core/ssh.py b/atlas/atlas/core/ssh.py index ae1206605..f1c8da6ef 100644 --- a/atlas/atlas/core/ssh.py +++ b/atlas/atlas/core/ssh.py @@ -26,7 +26,24 @@ def is_success(self) -> bool: class SSHRunner: """Run shell scripts on one SSH host.""" - def __init__(self, host: str, port: int = 22, user: str = "root") -> None: + options = ( + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "GlobalKnownHostsFile=/dev/null", + "-o", + "LogLevel=ERROR", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=30", + ) + + def __init__( + self, host: str, port: int = 22, user: str = "root", proxy_command: str | None = None + ) -> None: if not host: raise ValueError("SSH host is required") if not 1 <= port <= 65_535: @@ -37,6 +54,7 @@ def __init__(self, host: str, port: int = 22, user: str = "root") -> None: self.host = host self.port = port self.user = user + self.proxy_command = proxy_command def run_command( self, @@ -75,18 +93,8 @@ def _run( "ssh", "-p", str(self.port), - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - "GlobalKnownHostsFile=/dev/null", - "-o", - "LogLevel=ERROR", - "-o", - "BatchMode=yes", - "-o", - "ConnectTimeout=30", + *self.options, + *(("-o", f"ProxyCommand={self.proxy_command}") if self.proxy_command else ()), f"{self.user}@{self.host}", "bash -s", ], @@ -101,6 +109,10 @@ def _run( output = self._read_output(process, timeout_seconds, on_output) return SSHResult(output=output, exit_code=process.wait()) + def get_proxy_command(self, command: str) -> str: + """Return an SSH command line that runs one command on this host, for use as a ProxyCommand.""" + return shlex.join(["ssh", "-p", str(self.port), *self.options, f"{self.user}@{self.host}", command]) + @staticmethod def _read_output( process: subprocess.Popen[bytes], timeout_seconds: int, on_output: Callable[[str], None] | None @@ -149,7 +161,12 @@ def get_script_source(script: str) -> str: def wait_for_server( - *, host: str, users: tuple[str, ...], timeout_seconds: int, poll_interval_seconds: int + *, + host: str, + users: tuple[str, ...], + timeout_seconds: int, + poll_interval_seconds: int, + proxy_command: str | None = None, ) -> str: """Return the first SSH user that becomes available on a server.""" if not users: @@ -162,15 +179,19 @@ def wait_for_server( deadline = monotonic() + timeout_seconds while monotonic() < deadline: for user in users: - if _ssh_is_available(host, user): + if _ssh_is_available(host, user, proxy_command): return user sleep(poll_interval_seconds) raise TimeoutError(f"SSH did not become ready within {timeout_seconds} seconds") -def _ssh_is_available(host: str, user: str) -> bool: +def _ssh_is_available(host: str, user: str, proxy_command: str | None) -> bool: try: - return SSHRunner(host, user=user).run_command("true", timeout_seconds=10).is_success + return ( + SSHRunner(host, user=user, proxy_command=proxy_command) + .run_command("true", timeout_seconds=10) + .is_success + ) except OSError, subprocess.TimeoutExpired: return False diff --git a/atlas/atlas/core/test_artifacts.py b/atlas/atlas/core/test_artifacts.py index e0f199df1..c40ab395f 100644 --- a/atlas/atlas/core/test_artifacts.py +++ b/atlas/atlas/core/test_artifacts.py @@ -38,17 +38,37 @@ def test_download_url_prefers_the_configured_base(self) -> None: """A host cannot reach the site URL of a local bench.""" with ( patch.object(artifacts.frappe.db, "get_value", return_value="/files/metald-linux-amd64"), - patch.object(artifacts.frappe, "conf", SimpleNamespace(atlas_base_url="https://atlas.test/")), + patch.object( + artifacts.frappe, + "conf", + SimpleNamespace(atlas_internal_url=None, atlas_base_url="https://atlas.test/"), + ), ): url = artifacts.get_download_url("metald-file") self.assertEqual(url, "https://atlas.test/files/metald-linux-amd64") + def test_download_url_prefers_the_internal_listener(self) -> None: + """Host and tenant-0 VM downloads stay private when Atlas has a mesh listener.""" + conf = SimpleNamespace( + atlas_internal_url="http://[fdaa:1::ffff:ffff:ffff:ffff]:8000", + atlas_base_url="https://atlas.test", + ) + with ( + patch.object(artifacts.frappe.db, "get_value", return_value="/files/metald-linux-amd64"), + patch.object(artifacts.frappe, "conf", conf), + ): + url = artifacts.get_download_url("metald-file") + + self.assertEqual(url, "http://[fdaa:1::ffff:ffff:ffff:ffff]:8000/files/metald-linux-amd64") + def test_download_url_falls_back_to_the_site_url(self) -> None: """The request Host never names the address a host downloads from.""" with ( patch.object(artifacts.frappe.db, "get_value", return_value="/files/metald-linux-amd64"), - patch.object(artifacts.frappe, "conf", SimpleNamespace(atlas_base_url=None)), + patch.object( + artifacts.frappe, "conf", SimpleNamespace(atlas_internal_url=None, atlas_base_url=None) + ), patch.object( artifacts.frappe.utils, "get_url", return_value="http://atlas.localhost:8000" ) as get_url, diff --git a/atlas/atlas/core/test_mesh_address.py b/atlas/atlas/core/test_mesh_address.py index 8f85d2051..a0d7c3807 100644 --- a/atlas/atlas/core/test_mesh_address.py +++ b/atlas/atlas/core/test_mesh_address.py @@ -24,6 +24,12 @@ def test_a_wide_vm_number_fills_the_64_bit_field(self) -> None: self.assertEqual(address, "fdaa:1:0:7:ffff:ffff:ffff:ffff") + def test_a_tenant_zero_vm_cannot_take_the_atlas_address(self) -> None: + virtual_machine = frappe._dict(name="vm-18446744073709551615", tenant_id=0) + + with self.assertRaises(frappe.ValidationError): + get_virtual_machine_mesh_address(virtual_machine) + def test_a_vm_number_above_64_bits_is_refused(self) -> None: virtual_machine = frappe._dict(name="vm-18446744073709551616", tenant_id=7) diff --git a/atlas/atlas/core/test_ssh.py b/atlas/atlas/core/test_ssh.py index 5ef2959ef..1e152d051 100644 --- a/atlas/atlas/core/test_ssh.py +++ b/atlas/atlas/core/test_ssh.py @@ -31,6 +31,26 @@ def test_run_command_passes_host_port_and_data(self) -> None: process.stdin.write.call_args.args[0], b"export MESSAGE='hello world'\necho $MESSAGE" ) + def test_a_guest_connection_runs_through_the_host_proxy_command(self) -> None: + host = SSHRunner("fdab:1::7") + proxy_command = host.get_proxy_command("ip netns exec metal-vm-00001 nc 172.16.0.2 22") + runner = SSHRunner("vm-00001", proxy_command=proxy_command) + process = Mock() + process.wait.return_value = 0 + + with ( + patch("atlas.atlas.core.ssh.subprocess.Popen", return_value=process) as popen, + patch.object(runner, "_read_output", return_value=""), + ): + runner.run_command("true") + + arguments = popen.call_args.args[0] + self.assertIn(f"ProxyCommand={proxy_command}", arguments) + self.assertEqual(arguments[-2], "root@vm-00001") + self.assertTrue( + proxy_command.endswith("root@fdab:1::7 'ip netns exec metal-vm-00001 nc 172.16.0.2 22'") + ) + def test_run_script_loads_a_script_file(self) -> None: runner = SSHRunner("203.0.113.1") with patch.object(runner, "_run", return_value=SSHResult("", 0)) as run: diff --git a/atlas/atlas/doctype/atlas_settings/atlas_settings.json b/atlas/atlas/doctype/atlas_settings/atlas_settings.json index 1269e6004..d8d98677f 100644 --- a/atlas/atlas/doctype/atlas_settings/atlas_settings.json +++ b/atlas/atlas/doctype/atlas_settings/atlas_settings.json @@ -26,7 +26,6 @@ "metald_binary_x86_64_file", "metald_source_hash", "metald_binary_hash", - "use_public_ip_for_metald", "metal_tls_ca_private_key", "metal_tls_ca_certificate", "column_break_bgjv", @@ -42,11 +41,16 @@ "column_break_abbx", "central_jwks", "networking_tab", + "atlas_wireguard_peer_connectivity_section", + "wireguard_ip_address", + "column_break_uuos", + "wireguard_public_key", + "wireguard_private_key", "private_network_config_section", "private_network_cidr", - "is_unicast_network_enabled", "column_break_jzip", "private_network_mtu", + "is_unicast_network_enabled", "ipv6_router_section", "use_ipv6_router_for_auto_assignment", "ipv6_router_package_file", @@ -386,14 +390,6 @@ "label": "Metald Binary SHA-256", "read_only": 1 }, - { - "default": "0", - "description": "Enable when Atlas must reach Metald through the server's public IPv4 address.", - "fieldname": "use_public_ip_for_metald", - "fieldtype": "Check", - "label": "Reach Metald Over Public IP", - "show_description_on_click": 1 - }, { "fieldname": "metal_tls_ca_certificate", "fieldtype": "Password", @@ -823,13 +819,42 @@ "fieldname": "networking_tab", "fieldtype": "Tab Break", "label": "Networking" + }, + { + "fieldname": "wireguard_ip_address", + "fieldtype": "Data", + "label": "WireGuard IP Address", + "read_only": 1 + }, + { + "fieldname": "wireguard_public_key", + "fieldtype": "Data", + "label": "WireGuard Public Key", + "read_only": 1 + }, + { + "fieldname": "wireguard_private_key", + "fieldtype": "Password", + "hidden": 1, + "label": "WireGuard Private Key", + "read_only": 1 + }, + { + "description": "Atlas's own wireguard config. Use to connect to all other hosts.", + "fieldname": "atlas_wireguard_peer_connectivity_section", + "fieldtype": "Section Break", + "label": "Wireguard Config" + }, + { + "fieldname": "column_break_uuos", + "fieldtype": "Column Break" } ], "grid_page_length": 50, "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2026-10-01 00:00:00.000000", + "modified": "2026-10-02 02:05:36.893228", "modified_by": "Administrator", "module": "Atlas", "name": "Atlas Settings", diff --git a/atlas/atlas/doctype/atlas_settings/atlas_settings.py b/atlas/atlas/doctype/atlas_settings/atlas_settings.py index 7c0f95179..e6d7a9658 100644 --- a/atlas/atlas/doctype/atlas_settings/atlas_settings.py +++ b/atlas/atlas/doctype/atlas_settings/atlas_settings.py @@ -127,7 +127,6 @@ class AtlasSettings(Document): sleepy_vm_overcommit_factor: DF.Float use_dedicated_sleepy_vm_hosts: DF.Check use_ipv6_router_for_auto_assignment: DF.Check - use_public_ip_for_metald: DF.Check wg_mesh_binary_hash: DF.Data | None wg_mesh_binary_x86_64_file: DF.Link | None wg_mesh_source_hash: DF.Data | None @@ -135,6 +134,9 @@ class AtlasSettings(Document): wildcard_tls_certificate: DF.Password | None wildcard_tls_expires_on: DF.Datetime | None wildcard_tls_private_key: DF.Password | None + wireguard_ip_address: DF.Data | None + wireguard_private_key: DF.Password | None + wireguard_public_key: DF.Data | None # end: auto-generated types @property @@ -164,11 +166,11 @@ def issuer(self) -> str: @property def jwks_url(self) -> str: - """Return the public regional JSON Web Key Set URL.""" + """Return the regional JSON Web Key Set URL that the tenant-0 services fetch.""" + from atlas.atlas.core.artifacts import get_internal_base_url from atlas.auth.jwks import JWKS_PATH - base_url = frappe.conf.atlas_base_url or frappe.utils.get_url(allow_header_override=False) - return f"{base_url.rstrip('/')}{JWKS_PATH}" + return f"{get_internal_base_url().rstrip('/')}{JWKS_PATH}" @cached_property def server_provider_controller(self) -> "ServerProvider": diff --git a/atlas/atlas/doctype/ssh_task/README.md b/atlas/atlas/doctype/ssh_task/README.md index b85cde18d..30fff1afc 100644 --- a/atlas/atlas/doctype/ssh_task/README.md +++ b/atlas/atlas/doctype/ssh_task/README.md @@ -4,7 +4,9 @@ ## Target -Set `target_type` to `Metal Server` or `Virtual Machine`. Set `target` to the target document name. The task reads the target document's `ssh_host` property for the connection address. +Set `target_type` to `Metal Server` or `Virtual Machine`. Set `target` to the target document name. The task connects to the target's `ssh_host` through the command that the target's `get_ssh_proxy_command()` returns. + +A Metal Server returns no proxy command. Atlas connects to the host `wg0` address directly. A Virtual Machine returns an SSH command to its current host that runs `ip netns exec metal- nc 172.16.0.2 22`, so the guest needs no public address. See [Atlas access to hosts](../../../../docs/region/host-access.md#ssh). `SSH Task` stores no credentials. It uses the identity of the Atlas host. Make the target reachable before you create the task. diff --git a/atlas/atlas/doctype/ssh_task/ssh_task.py b/atlas/atlas/doctype/ssh_task/ssh_task.py index 844a2cc85..d50b12ea9 100644 --- a/atlas/atlas/doctype/ssh_task/ssh_task.py +++ b/atlas/atlas/doctype/ssh_task/ssh_task.py @@ -175,7 +175,7 @@ def execute(self) -> SSHResult: self._mark_running() try: target = frappe.get_doc(self.target_type, self.target) - runner = SSHRunner(target.ssh_host, self.port, self.ssh_user) + runner = SSHRunner(target.ssh_host, self.port, self.ssh_user, target.get_ssh_proxy_command()) result = runner.run_command( self.script, data=self._environment(), diff --git a/atlas/atlas/doctype/ssh_task/test_ssh_task.py b/atlas/atlas/doctype/ssh_task/test_ssh_task.py index fcbdfc415..e8d3b20f6 100644 --- a/atlas/atlas/doctype/ssh_task/test_ssh_task.py +++ b/atlas/atlas/doctype/ssh_task/test_ssh_task.py @@ -191,12 +191,14 @@ def test_execute_reads_the_address_from_the_target(self) -> None: with ( patch( "atlas.atlas.doctype.ssh_task.ssh_task.frappe.get_doc", - return_value=SimpleNamespace(ssh_host="203.0.113.7"), + return_value=SimpleNamespace( + ssh_host="vm-00001", get_ssh_proxy_command=lambda: "ssh host nc" + ), ) as get_doc, patch("atlas.atlas.doctype.ssh_task.ssh_task.SSHRunner", return_value=runner) as ssh_runner, ): SSHTask.execute(task) get_doc.assert_called_once_with("Virtual Machine", "vm-00001") - ssh_runner.assert_called_once_with("203.0.113.7", 22, "root") + ssh_runner.assert_called_once_with("vm-00001", 22, "root", "ssh host nc") runner.run_command.assert_called_once() diff --git a/atlas/commands.py b/atlas/commands.py index 4b1960eb0..ab09ee6ec 100644 --- a/atlas/commands.py +++ b/atlas/commands.py @@ -21,6 +21,9 @@ ) from atlas.atlas.core.setup import AtlasSetup, AtlasSetupConfiguration from atlas.atlas.object_storage import ObjectStorageError +from atlas.metal_server.core.atlas_peer import AtlasPeer +from atlas.metal_server.core.development_gateway import DevelopmentGateway +from atlas.metal_server.doctype.metal_server.metal_server import MetalServer from atlas.service.core.service_package import SERVICE_PACKAGES from atlas.vm.core.image_builder import build_ubuntu_image, publish_ubuntu_image @@ -202,4 +205,76 @@ def is_image_available(site: str, title: str, architecture: str) -> bool: frappe.destroy() -commands = [configure_atlas, build_metald, build_wg_mesh, build_service_packages, build_ubuntu_base_image] +@click.command("configure-atlas-wireguard") +@pass_context +def configure_atlas_wireguard(context: CliCtxObj) -> None: + """Create the Atlas wg0 identity once and write its wg-quick file.""" + if not context.sites: + raise SiteNotSpecifiedError + + for site in context.sites: + try: + frappe.init(site) + frappe.connect() + atlas_peer = AtlasPeer() + atlas_peer.ensure_identity() + atlas_peer.write_config() + frappe.db.commit() # nosemgrep + click.echo(f"{site}: {atlas_peer.settings.wireguard_ip_address} {atlas_peer.config_path}") + finally: + frappe.destroy() + + +@click.command("deploy-dev-gateway") +@click.argument("metal_server") +@click.option( + "--ssh-host", + help="Reach a host that has no Atlas link yet, for example the first host by its public IPv4.", +) +@pass_context +def deploy_dev_gateway(context: CliCtxObj, metal_server: str, ssh_host: str | None = None) -> None: + """Run the development gateway on a Metal host and write the local wg-quick file.""" + if not context.sites: + raise SiteNotSpecifiedError + + for site in context.sites: + try: + frappe.init(site) + frappe.connect() + config_path = DevelopmentGateway(frappe.get_doc("Metal Server", metal_server)).install(ssh_host) + frappe.db.commit() # nosemgrep + click.echo(f"{site}: {config_path}") + finally: + frappe.destroy() + + +@click.command("import-metal-server") +@click.argument("provider_server_id") +@click.option("--storage-pool-device", help="Device or disk image file for the storage pool.") +@pass_context +def import_metal_server(context: CliCtxObj, provider_server_id: str, storage_pool_device: str | None) -> None: + """Add a provider server that was created outside Atlas, or continue its setup.""" + if not context.sites: + raise SiteNotSpecifiedError + + for site in context.sites: + try: + frappe.init(site) + frappe.connect() + server = MetalServer.import_from_provider(provider_server_id, storage_pool_device) + frappe.db.commit() # nosemgrep + click.echo(f"{site}: Metal Server {server.name} ({server.title}) is {server.status}") + finally: + frappe.destroy() + + +commands = [ + configure_atlas, + build_metald, + build_wg_mesh, + build_service_packages, + build_ubuntu_base_image, + configure_atlas_wireguard, + deploy_dev_gateway, + import_metal_server, +] diff --git a/atlas/hooks.py b/atlas/hooks.py index 940e51cad..c57f1ed65 100644 --- a/atlas/hooks.py +++ b/atlas/hooks.py @@ -212,6 +212,7 @@ "atlas.metal_server.doctype.public_ip_allocation.public_ip_allocation.enqueue_pending_allocation_reconciliation", "atlas.metal_server.doctype.public_ip_allocation.public_ip_allocation.enqueue_allocation_moves", "atlas.metal_server.usage.enqueue_server_syncs", + "atlas.metal_server.core.atlas_peer.write_atlas_peer_config", ], "* * * * * */30": [ "atlas.vm.core.vm_image_transfer.enqueue_pending_virtual_machine_image_transfers", diff --git a/atlas/metal_server/SPEC.md b/atlas/metal_server/SPEC.md index 984cc2321..6a96f4aaa 100644 --- a/atlas/metal_server/SPEC.md +++ b/atlas/metal_server/SPEC.md @@ -2,7 +2,7 @@ [Atlas app specification](../SPEC.md) -Behavior: [provisioning](../../docs/region/index.md), [hosts](../../docs/region/hosts-and-providers.md), [host sync](../../docs/region/host-sync.md), and [public IPs](../../docs/networking/public-ips.md). A Metal Server is a provider host that runs Metal. +Behavior: [provisioning](../../docs/region/index.md), [hosts](../../docs/region/hosts-and-providers.md), [Atlas access to hosts](../../docs/region/host-access.md), [host sync](../../docs/region/host-sync.md), and [public IPs](../../docs/networking/public-ips.md). A Metal Server is a provider host that runs Metal. ## Types @@ -10,7 +10,10 @@ Behavior: [provisioning](../../docs/region/index.md), [hosts](../../docs/region/ |---|---| | `MetalServer` (DocType) | Lifecycle, permissions, whitelisted API | | `provisioning` | Phase order, progress, failure logs | -| `host_installation` | Installs Metal on the host | +| `host_installation` | Installs Metal and the host firewall on the host | +| `AtlasPeer` | The Atlas wg0 identity and its `atlas0.conf` peer file | +| `DevelopmentGateway` | Development only. atlas-vm gateway mode on one host and the local `atlas-gateway.conf` | +| `MetalServer.import_from_provider` | Adds a provider server that Atlas did not create. Idempotent by provider server ID. | | `disk_inventory` | Block devices to Metal Server Disk rows | | `catalog_sync` | Size and Image catalogs from the provider | | `host_inspection` | Generic host registration. See [providers](../../docs/region/provider-guide.md#generic-provider). | @@ -25,7 +28,9 @@ Host commands use [SSH Task](../atlas/doctype/ssh_task/) from the Atlas module. - Provisioning commits after each phase. Every phase must be safe to repeat. - Creation reuses the stored identity key, so a lost provider response cannot create a second host. -- The provider returns the `metald` listen address and the storage pool device. Host installation never searches for a disk. +- `metald` listens only on the host WireGuard address. The provider returns the storage pool device. Host installation never searches for a disk. +- `ssh_host` is the host WireGuard address once the host has a WireGuard key. Only setup before that uses the public address. +- The Atlas peer lives in the host `wg0.conf`, not in host sync. Metal removes only the peers it added. - Certificate renewal restarts `metal.service` and shares the metald job lock with install and upgrade. - Placement reads Metal Server Usage rows that `usage` writes after each `POST /v1/sync`. - Atlas WG Mesh identifies a peer by `private_network_mac_address`. Sync writes it only when it changes. diff --git a/atlas/metal_server/core/atlas_peer.py b/atlas/metal_server/core/atlas_peer.py new file mode 100644 index 000000000..0e8001d18 --- /dev/null +++ b/atlas/metal_server/core/atlas_peer.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import base64 +import os +import re +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, cast + +import frappe +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from frappe import _ + +from atlas.atlas.core.mesh_address import ( + WIREGUARD_PREFIX, + get_atlas_mesh_address, + get_region_mesh_address_prefix, + get_virtual_machine_mesh_address, +) + +if TYPE_CHECKING: + from atlas.atlas.doctype.atlas_settings.atlas_settings import AtlasSettings + +PEER_KEEPALIVE_SECONDS = 25 +WIREGUARD_KEY = re.compile(r"[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=") + + +class AtlasPeer: + """Own the Atlas wg0 identity and the wg-quick file that a root timer applies.""" + + interface = "atlas0" + + def __init__(self, settings: "AtlasSettings | None" = None) -> None: + self.settings = settings or cast("AtlasSettings", frappe.get_single("Atlas Settings")) + + @property + def directory(self) -> Path: + """Return the private directory that holds the wg-quick files.""" + return Path(frappe.get_site_path("private", "wireguard")).resolve() + + @property + def config_path(self) -> Path: + """Return the wg-quick file that the timer applies.""" + return self.directory / f"{self.interface}.conf" + + def ensure_identity(self) -> None: + """Create the Atlas key once in Atlas Settings, so site backups keep it, and store the mesh address.""" + address = get_atlas_mesh_address(self.settings.region_id) + if self.settings.wireguard_ip_address != address: + self.settings.db_set("wireguard_ip_address", address) + + if self.settings.wireguard_public_key: + return + + self.settings.wireguard_private_key = generate_private_key() + self.settings.wireguard_public_key = get_public_key(self.settings.wireguard_private_key) + self.settings.save(ignore_permissions=True, ignore_version=True) + + def write_config(self) -> bool: + """Write the wg-quick file with one peer for each host. Return whether it changed.""" + if not self.settings.wireguard_public_key: + return False + + self.ensure_identity() + config = self.get_config() + if self.config_path.is_file() and self.config_path.read_text() == config: + return False + + write_private_file(self.config_path, config) + return True + + def get_config(self) -> str: + """Return the wg-quick file content.""" + region_id = self.settings.region_id + sections = [ + "[Interface]", + f"PrivateKey = {self.settings.get_password('wireguard_private_key')}", + f"Address = {self.settings.wireguard_ip_address}/128", + # wg syncconf adds no routes. + f"PostUp = ip -6 route replace {WIREGUARD_PREFIX:x}:{region_id:x}::/32 dev %i", + f"PostUp = ip -6 route replace {get_region_mesh_address_prefix(region_id)}::/64 dev %i", + ] + if frappe.conf.atlas_wireguard_mtu: + sections.append(f"MTU = {int(frappe.conf.atlas_wireguard_mtu)}") + virtual_machines = self.get_tenant_zero_virtual_machines() + for host in self.get_host_peers(): + allowed_addresses = [host.wireguard_ip_address, *virtual_machines.get(host.name, [])] + sections += [ + "", + "[Peer]", + f"PublicKey = {host.wireguard_public_key}", + f"AllowedIPs = {', '.join(f'{address}/128' for address in allowed_addresses)}", + f"Endpoint = {host.endpoint_address}:{host.port}", + f"PersistentKeepalive = {PEER_KEEPALIVE_SECONDS}", + ] + return "\n".join(sections) + "\n" + + def get_tenant_zero_virtual_machines(self) -> dict[str, list[str]]: + """Return the mesh addresses of tenant-0 VMs by host.""" + addresses: dict[str, list[str]] = {} + for virtual_machine in frappe.get_all( + "Virtual Machine", + # "is set" never matches the uuid column server. A VM without a server has no host peer. + filters={"tenant_id": 0, "is_draft": 0}, + fields=["name", "tenant_id", "server"], + order_by="name asc", + ): + addresses.setdefault(virtual_machine.server, []).append( + get_virtual_machine_mesh_address(virtual_machine, self.settings.region_id) + ) + return addresses + + def get_host_peers(self) -> list[frappe._dict]: + """Return every configured host. Reject a host that reuses the Atlas key.""" + hosts = frappe.get_all( + "Metal Server", + filters={ + "status": ["!=", "Deleted"], + "wireguard_public_key": ["is", "set"], + "wireguard_ip_address": ["is", "set"], + "private_ipv4_address": ["is", "set"], + }, + fields=[ + "name", + "wireguard_public_key", + "wireguard_ip_address", + "port", + "private_ipv4_address as endpoint_address", + ], + order_by="name asc", + ) + for host in hosts: + if self.settings.wireguard_public_key == host.wireguard_public_key: + frappe.throw(_("Metal Server {0} uses the Atlas WireGuard public key.").format(host.name)) + return hosts + + +def write_atlas_peer_config() -> None: + """Refresh the Atlas wg-quick file.""" + AtlasPeer().write_config() + + +def generate_private_key() -> str: + """Return a new WireGuard private key.""" + private_bytes = X25519PrivateKey.generate().private_bytes( + serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption() + ) + return base64.b64encode(private_bytes).decode() + + +def get_public_key(private_key: str) -> str: + """Return the public key of one WireGuard private key.""" + key = X25519PrivateKey.from_private_bytes(base64.b64decode(private_key)) + public_bytes = key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return base64.b64encode(public_bytes).decode() + + +def write_private_file(path: Path, content: str) -> None: + """Replace one file that only the site user can read.""" + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + file_descriptor, temporary_path = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + with os.fdopen(file_descriptor, "w") as temporary: + temporary.write(content) + os.replace(temporary_path, path) diff --git a/atlas/metal_server/core/development_gateway.py b/atlas/metal_server/core/development_gateway.py new file mode 100644 index 000000000..c9f79e16e --- /dev/null +++ b/atlas/metal_server/core/development_gateway.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING, cast + +import frappe +from frappe import _ + +from atlas.atlas.core.ssh import SSHRunner +from atlas.metal_server.core.atlas_peer import ( + PEER_KEEPALIVE_SECONDS, + WIREGUARD_KEY, + AtlasPeer, + generate_private_key, + get_public_key, + write_private_file, +) + +if TYPE_CHECKING: + from atlas.atlas.doctype.atlas_settings.atlas_settings import AtlasSettings + from atlas.metal_server.doctype.metal_server.metal_server import MetalServer + + +class DevelopmentGateway: + """Run atlas-vm as the development gateway on one Metal host, and write the local link to it.""" + + interface = "atlas-gateway" + # atlas-vm owns these values. + listen_port = 51821 + developer_address = "172.16.100.3" + deploy_command = """set -eu +if [ -f /var/lib/atlas-vm/atlas-vm.toml ] && ! grep -qxF '[gateway]' /var/lib/atlas-vm/atlas-vm.toml; then + echo "This host runs an Atlas VM." >&2 + exit 1 +fi +install -d -m 0700 /var/lib/atlas-vm +umask 077 +printf '%s' "$ATLAS_VM" > /usr/local/bin/atlas-vm +chmod 0755 /usr/local/bin/atlas-vm +printf '%s' "$ATLAS_VM_CONFIG" > /var/lib/atlas-vm/atlas-vm.toml +unset ATLAS_VM ATLAS_VM_CONFIG +if systemctl is-active --quiet atlas-pilot-vm.service; then + atlas-vm setup +else + atlas-vm create --config /var/lib/atlas-vm/atlas-vm.toml +fi""" + + def __init__(self, server: "MetalServer", settings: "AtlasSettings | None" = None) -> None: + self.server = server + self.settings = settings or cast("AtlasSettings", frappe.get_single("Atlas Settings")) + self.atlas_peer = AtlasPeer(self.settings) + + @property + def private_key_path(self) -> Path: + """Return the developer key for the outer link.""" + return self.atlas_peer.directory / f"{self.interface}.key" + + @property + def config_path(self) -> Path: + """Return the wg-quick file of the outer link.""" + return self.atlas_peer.directory / f"{self.interface}.conf" + + def install(self, ssh_host: str | None = None) -> Path: + """Deploy the gateway over SSH and write the local wg-quick file.""" + if not self.private_key_path.is_file(): + write_private_file(self.private_key_path, generate_private_key() + "\n") + + configuration = ( + f"[vm]\nvcpu_count = 1\nmemory_mib = 1024\ndisk_gib = 8\n\n" + f'[atlas]\nprivate_network_cidr = "{self.settings.private_network_cidr}"\n\n' + f'[gateway]\ndeveloper_public_key = "{get_public_key(self.private_key_path.read_text())}"\n' + ) + script = Path(frappe.get_app_path("atlas")).parent / "scripts/atlas-vm/atlas_vm.py" + result = SSHRunner(ssh_host or self.server.ssh_host).run_command( + self.deploy_command, + data={"ATLAS_VM": script.read_text(), "ATLAS_VM_CONFIG": configuration}, + timeout_seconds=1800, + ) + match = re.search(r"gateway public key: (\S+)", result.output) + if not result.is_success or not match or not WIREGUARD_KEY.fullmatch(match.group(1)): + frappe.throw(_("The gateway setup failed: {0}").format(result.output.strip()[-500:])) + + write_private_file(self.config_path, self.get_config(match.group(1))) + return self.config_path + + def get_config(self, gateway_public_key: str) -> str: + """Return the outer link. It carries only the provider private network.""" + return ( + "\n".join( + [ + "[Interface]", + f"PrivateKey = {self.private_key_path.read_text().strip()}", + f"Address = {self.developer_address}/32", + "", + "[Peer]", + f"PublicKey = {gateway_public_key}", + f"AllowedIPs = {self.settings.private_network_cidr}", + f"Endpoint = {self.server.public_ipv4_address}:{self.listen_port}", + f"PersistentKeepalive = {PEER_KEEPALIVE_SECONDS}", + ] + ) + + "\n" + ) diff --git a/atlas/metal_server/core/host_installation.py b/atlas/metal_server/core/host_installation.py index e6a57e7d2..2b2db8d2a 100644 --- a/atlas/metal_server/core/host_installation.py +++ b/atlas/metal_server/core/host_installation.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ipaddress from typing import TYPE_CHECKING from uuid import UUID @@ -8,9 +7,11 @@ from frappe import _ from atlas.atlas.core.artifacts import get_download_url +from atlas.atlas.core.mesh_address import get_wireguard_ip_address from atlas.atlas.core.ssh import SSHRunner from atlas.atlas.core.tls.metal import atlas_client_identity, ensure_server_certificate from atlas.atlas.doctype.ssh_task.ssh_task import SSHTask +from atlas.metal_server.core.atlas_peer import WIREGUARD_KEY if TYPE_CHECKING: from atlas.atlas.core.ssh import SSHResult @@ -21,13 +22,9 @@ WIREGUARD_CONFIGURE_TIMEOUT_SECONDS = 300 METALD_INSTALL_TIMEOUT_SECONDS = 1_200 STORAGE_INSTALL_TIMEOUT_SECONDS = 600 +FIREWALL_INSTALL_TIMEOUT_SECONDS = 120 METALD_SETUP_TIMEOUT_SECONDS = 3_600 -MESH_PREFIX = 0xFDAB -# The prefix and the region take the first 32 bits of the address, so the low 96 -# bits of the server UUID are the host part. -MESH_HOST_MASK = (1 << 96) - 1 - class HostInstallation: """Own the Atlas software installation on one server.""" @@ -41,6 +38,11 @@ def configure_wireguard(self) -> None: frappe.throw(_("Metal Server {0} needs a private network interface.").format(self.server.name)) self.set_wireguard_ip_address() + settings = self.server.settings + if not settings.wireguard_public_key: + frappe.throw( + _("Atlas has no WireGuard identity. Run pilot --site SITE configure-atlas-wireguard first.") + ) result = SSHTask.create_for_script_file( target_type=self.server.doctype, target=self.server.name, @@ -49,6 +51,8 @@ def configure_wireguard(self) -> None: "WIREGUARD_ADDRESS": self.server.wireguard_ip_address, "WIREGUARD_LISTEN_PORT": self.server.port, "MESH_UPLINK_INTERFACE": self.server.private_network_interface, + "ATLAS_WIREGUARD_ADDRESS": settings.wireguard_ip_address, + "ATLAS_WIREGUARD_PUBLIC_KEY": settings.wireguard_public_key, }, timeout_seconds=WIREGUARD_CONFIGURE_TIMEOUT_SECONDS, run_in_background=False, @@ -58,8 +62,9 @@ def configure_wireguard(self) -> None: _("Could not configure WireGuard on server {0}.").format(self.server.name), result ) - public_key = result.output.partition("===PUBLIC_KEY_START===")[2] - public_key = public_key.partition("===PUBLIC_KEY_END===")[0].strip() + # SSH stderr can land between the markers. + section = result.output.partition("===PUBLIC_KEY_START===")[2].partition("===PUBLIC_KEY_END===")[0] + public_key = next((line for line in section.split() if WIREGUARD_KEY.fullmatch(line)), "") if not public_key: frappe.throw(_("Metal Server {0} reported no WireGuard public key.").format(self.server.name)) self.server.db_set("wireguard_public_key", public_key) @@ -78,15 +83,6 @@ def install_metal(self) -> None: self.install_storage() self.install_tls_credentials() - listen_address = settings.server_provider_controller.metald_listen_address(self.server) - try: - listen_address = str(ipaddress.IPv4Address(listen_address)) - except ipaddress.AddressValueError: - frappe.throw( - _("Metal Server {0} has an invalid address for the selected metald endpoint.").format( - self.server.name - ), - ) result = SSHTask.create_for_script_file( target_type=self.server.doctype, target=self.server.name, @@ -96,10 +92,12 @@ def install_metal(self) -> None: "METALD_SHA256": settings.metald_binary_hash, "WG_MESH_DOWNLOAD_URL": get_download_url(settings.wg_mesh_binary_x86_64_file), "WG_MESH_SHA256": settings.wg_mesh_binary_hash, - "LISTEN_ADDRESS": f"{listen_address}:9000", + "LISTEN_ADDRESS": f"[{self.server.wireguard_ip_address}]:9000", "ATLAS_COMMON_NAME": atlas_client_identity(settings), "COORDINATION_LISTEN_ADDRESS": f"[{self.server.wireguard_ip_address}]:9001", "MESH_UPLINK_INTERFACE": self.server.private_network_interface, + "PRIVATE_NETWORK_CIDR": settings.private_network_cidr, + "ATLAS_MESH_ADDRESS": settings.wireguard_ip_address, }, timeout_seconds=METALD_INSTALL_TIMEOUT_SECONDS, run_in_background=False, @@ -109,6 +107,8 @@ def install_metal(self) -> None: _("Could not install metald on server {0}.").format(self.server.name), result ) + self.install_host_firewall() + def install_storage(self) -> None: """Create the storage pool and mount host state on it before any file is written there.""" result = SSHTask.create_for_script_file( @@ -116,9 +116,9 @@ def install_storage(self) -> None: target=self.server.name, script_path="install-metal-storage.sh", environment={ - "STORAGE_POOL_DEVICE": self.server.settings.server_provider_controller.storage_pool_device( + "STORAGE_POOL_DEVICE": self.server.settings.server_provider_controller.get_storage_pool_device( self.server - ), + ) }, timeout_seconds=STORAGE_INSTALL_TIMEOUT_SECONDS, run_in_background=False, @@ -128,6 +128,27 @@ def install_storage(self) -> None: _("Could not prepare storage on server {0}.").format(self.server.name), result ) + def install_host_firewall(self) -> None: + """Admit host SSH and Metal calls only through wg0, and keep guests off the private network.""" + settings = self.server.settings + result = SSHTask.create_for_script_file( + target_type=self.server.doctype, + target=self.server.name, + script_path="install-host-firewall.sh", + environment={ + "ATLAS_WIREGUARD_ADDRESS": settings.wireguard_ip_address, + "MESH_UPLINK_INTERFACE": self.server.private_network_interface, + "PRIVATE_NETWORK_CIDR": settings.private_network_cidr, + "WIREGUARD_LISTEN_PORT": self.server.port, + }, + timeout_seconds=FIREWALL_INSTALL_TIMEOUT_SECONDS, + run_in_background=False, + ).result + if not result or not result.is_success: + throw_script_failure( + _("Could not install the host firewall on server {0}.").format(self.server.name), result + ) + def install_tls_credentials(self) -> None: """Issue a current node certificate and install it. The script restarts a running Metal.""" ca_certificate, certificate, private_key = ensure_server_certificate(self.server) @@ -176,12 +197,7 @@ def set_wireguard_ip_address(self) -> None: @property def wireguard_ip_address(self) -> str: """Return the host mesh address for this server.""" - region_id = self.server.settings.region_id - if not 0 <= region_id <= 0xFFFF: - frappe.throw(_("Atlas Settings region ID must fit in one IPv6 field.")) - - host = UUID(self.server.name).int & MESH_HOST_MASK - return str(ipaddress.IPv6Address((MESH_PREFIX << 112) | (region_id << 96) | host)) + return get_wireguard_ip_address(UUID(self.server.name), self.server.settings.region_id) def throw_script_failure(message: str, result: "SSHResult | None") -> None: diff --git a/atlas/metal_server/core/provisioning.py b/atlas/metal_server/core/provisioning.py index 9d94ac576..d3a9196a1 100644 --- a/atlas/metal_server/core/provisioning.py +++ b/atlas/metal_server/core/provisioning.py @@ -8,6 +8,7 @@ from atlas.atlas.core.server_providers.base import ProviderOperationError, ServerProvider from atlas.atlas.core.ssh import wait_for_server +from atlas.metal_server.core.atlas_peer import AtlasPeer from atlas.metal_server.core.host_installation import HostInstallation if TYPE_CHECKING: @@ -31,6 +32,7 @@ class ServerProvisioner: "wireguard_public_key", ) ssh_timeout_seconds = 1_000 + wireguard_timeout_seconds = 120 ssh_poll_interval_seconds = 1 def __init__(self, server: "MetalServer", provider: ServerProvider | None = None) -> None: @@ -70,6 +72,7 @@ def steps(self) -> tuple[tuple[str, Callable[[], None]], ...]: ("secure-shell", self.wait_for_root_ssh), ("provider-network", lambda: self.provider.configure_server_network(self.server)), ("wireguard", self.host_installation.configure_wireguard), + ("wireguard-link", self.wait_for_wireguard_ssh), ("metal", self.host_installation.install_metal), ) @@ -83,13 +86,13 @@ def run_step(self, phase: str, operation: Callable[[], None]) -> None: self.save_progress() def wait_for_root_ssh(self) -> None: - """Wait until the root Secure Shell account is available.""" - if not self.server.public_ipv4_address: + """Wait until the root Secure Shell account is available. A retry after WireGuard setup uses wg0.""" + if not self.server.wireguard_public_key and not self.server.public_ipv4_address: raise ProviderOperationError("Server has no public IPv4 address") try: user = wait_for_server( - host=self.server.public_ipv4_address, + host=self.server.ssh_host, users=self.provider.ssh_users, timeout_seconds=self.ssh_timeout_seconds, poll_interval_seconds=self.ssh_poll_interval_seconds, @@ -102,7 +105,7 @@ def wait_for_root_ssh(self) -> None: self.provider.promote_ssh_user(self.server, user) try: root_user = wait_for_server( - host=self.server.public_ipv4_address, + host=self.server.ssh_host, users=("root",), timeout_seconds=self.ssh_timeout_seconds, poll_interval_seconds=self.ssh_poll_interval_seconds, @@ -118,6 +121,23 @@ def wait_for_root_ssh(self) -> None: is_retryable=True, ) + def wait_for_wireguard_ssh(self) -> None: + """Wait until root Secure Shell answers on the host wg0 address.""" + # Publish the new host peer now instead of waiting for the scheduler. + AtlasPeer(self.server.settings).write_config() + try: + wait_for_server( + host=self.server.ssh_host, + users=("root",), + timeout_seconds=self.wireguard_timeout_seconds, + poll_interval_seconds=self.ssh_poll_interval_seconds, + ) + except TimeoutError as error: + raise ProviderOperationError( + f"Atlas cannot reach {self.server.ssh_host} through wg0. Check that {AtlasPeer.interface} holds this host.", + is_retryable=True, + ) from error + def save_progress(self) -> None: """Store the current setup fields and commit the setup transaction.""" self.server.db_set({field: self.server.get(field) for field in self.setup_fields}) diff --git a/atlas/metal_server/core/test_atlas_peer.py b/atlas/metal_server/core/test_atlas_peer.py new file mode 100644 index 000000000..6be71b59d --- /dev/null +++ b/atlas/metal_server/core/test_atlas_peer.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import base64 +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, PropertyMock, patch + +import frappe +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from frappe.tests import UnitTestCase + +from atlas.metal_server.core import atlas_peer as atlas_peer_module +from atlas.metal_server.core.atlas_peer import AtlasPeer + +HOST = frappe._dict( + name="host-1", + wireguard_public_key="host-key", + wireguard_ip_address="fdab:1::7", + port=51820, + endpoint_address="10.1.0.7", +) +PROXY = frappe._dict(name="vm-0000082", tenant_id=0, server="host-1") + + +class TestAtlasPeer(UnitTestCase): + def setUp(self) -> None: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.directory = Path(directory.name) / "wireguard" + self.passwords: dict[str, str] = {} + patcher = patch.object(AtlasPeer, "directory", new_callable=PropertyMock, return_value=self.directory) + patcher.start() + self.addCleanup(patcher.stop) + + def test_identity_is_a_stored_private_key_and_a_tenant_zero_address(self) -> None: + """The key lives in Atlas Settings, so a site backup can restore it.""" + atlas_peer = AtlasPeer(self.settings()) + + atlas_peer.ensure_identity() + + private_key = X25519PrivateKey.from_private_bytes( + base64.b64decode(self.passwords["wireguard_private_key"]) + ) + public_key = private_key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + self.assertEqual(atlas_peer.settings.wireguard_public_key, base64.b64encode(public_key).decode()) + self.assertEqual(atlas_peer.settings.wireguard_ip_address, "fdaa:1::ffff:ffff:ffff:ffff") + + def test_identity_is_created_once(self) -> None: + atlas_peer = AtlasPeer(self.settings()) + atlas_peer.ensure_identity() + public_key = atlas_peer.settings.wireguard_public_key + + atlas_peer.ensure_identity() + + self.assertEqual(atlas_peer.settings.wireguard_public_key, public_key) + + def test_config_routes_hosts_and_their_tenant_zero_vms(self) -> None: + atlas_peer = AtlasPeer(self.settings()) + atlas_peer.ensure_identity() + + with self.records(): + config = atlas_peer.get_config() + + self.assertIn(f"PrivateKey = {self.passwords['wireguard_private_key']}\n", config) + self.assertIn("Address = fdaa:1::ffff:ffff:ffff:ffff/128", config) + self.assertIn("PostUp = ip -6 route replace fdab:1::/32 dev %i", config) + self.assertIn("PostUp = ip -6 route replace fdaa:1::/64 dev %i", config) + self.assertIn( + "[Peer]\nPublicKey = host-key\nAllowedIPs = fdab:1::7/128, fdaa:1::52/128\nEndpoint = 10.1.0.7:51820\n", + config, + ) + + def test_a_host_with_the_atlas_key_is_rejected(self) -> None: + atlas_peer = AtlasPeer(self.settings(wireguard_public_key="host-key")) + + with ( + patch.object(atlas_peer_module.frappe, "get_all", return_value=[HOST]), + self.assertRaisesRegex(frappe.ValidationError, "uses the Atlas WireGuard public key"), + ): + atlas_peer.get_host_peers() + + def test_an_unchanged_config_is_not_written_again(self) -> None: + atlas_peer = AtlasPeer(self.settings()) + atlas_peer.ensure_identity() + + with self.records(): + self.assertTrue(atlas_peer.write_config()) + self.assertFalse(atlas_peer.write_config()) + + self.assertEqual(atlas_peer.config_path.stat().st_mode & 0o777, 0o600) + + @staticmethod + @contextmanager + def records() -> Iterator[Mock]: + """Return one host and the proxy VM that runs on it.""" + + def get_all(doctype: str, **_arguments: object) -> list[frappe._dict]: + return [PROXY] if doctype == "Virtual Machine" else [HOST] + + with ( + patch.object(atlas_peer_module.frappe, "get_all", side_effect=get_all) as get_all_mock, + patch( + "atlas.atlas.core.mesh_address.frappe.get_single", return_value=SimpleNamespace(region_id=1) + ), + patch.object(atlas_peer_module.frappe, "conf", frappe._dict()), + ): + yield get_all_mock + + def settings(self, **values: object) -> SimpleNamespace: + settings = SimpleNamespace( + **{ + "doctype": "Atlas Settings", + "name": "Atlas Settings", + "region_id": 1, + "wireguard_ip_address": None, + "wireguard_public_key": None, + **values, + } + ) + settings.db_set = lambda field, value=None: settings.__dict__.update({field: value}) + settings.get_password = lambda field: self.passwords[field] + settings.save = lambda **_: self.passwords.update( + wireguard_private_key=settings.wireguard_private_key + ) + return settings diff --git a/atlas/metal_server/core/test_provisioning.py b/atlas/metal_server/core/test_provisioning.py index 59c1c7002..d18af35bd 100644 --- a/atlas/metal_server/core/test_provisioning.py +++ b/atlas/metal_server/core/test_provisioning.py @@ -5,6 +5,7 @@ from frappe.tests import UnitTestCase +from atlas.atlas.core.server_providers.base import ProviderOperationError from atlas.metal_server.core.provisioning import ServerProvisioner @@ -20,6 +21,7 @@ def test_run_uses_the_safe_setup_order(self) -> None: provisioner.wait_for_root_ssh = Mock(side_effect=lambda: operations("secure-shell")) provider.configure_server_network.side_effect = lambda _server: operations("provider-network") provisioner.host_installation.configure_wireguard.side_effect = lambda: operations("wireguard") + provisioner.wait_for_wireguard_ssh = Mock(side_effect=lambda: operations("wireguard-link")) provisioner.host_installation.install_metal.side_effect = lambda: operations("metal") with patch("atlas.metal_server.core.provisioning.frappe.db", SimpleNamespace(commit=Mock())): @@ -33,6 +35,7 @@ def test_run_uses_the_safe_setup_order(self) -> None: "secure-shell", "provider-network", "wireguard", + "wireguard-link", "metal", ], ) @@ -67,6 +70,7 @@ def test_a_retry_runs_each_idempotent_step_again(self) -> None: provider = Mock() provisioner = ServerProvisioner(server, provider) provisioner.wait_for_root_ssh = Mock() + provisioner.wait_for_wireguard_ssh = Mock() provisioner.host_installation = Mock() with patch("atlas.metal_server.core.provisioning.frappe.db", SimpleNamespace(commit=Mock())): @@ -95,6 +99,43 @@ def test_provider_creation_failure_marks_the_pending_host_failed(self) -> None: self.assertEqual(server.status, "Failed") self.assertIn("provider-create", log_error.call_args.kwargs["title"]) + def test_wireguard_link_publishes_the_host_peer_then_waits_for_root_on_wg0(self) -> None: + """Setup must not wait for the scheduler to add the new host to atlas0.""" + provisioner = ServerProvisioner(self.server(), Mock()) + + with ( + patch("atlas.metal_server.core.provisioning.AtlasPeer") as atlas_peer, + patch("atlas.metal_server.core.provisioning.wait_for_server") as wait_for_server, + ): + provisioner.wait_for_wireguard_ssh() + + atlas_peer.return_value.write_config.assert_called_once_with() + self.assertEqual(wait_for_server.call_args.kwargs["host"], "fdab:1::1") + self.assertEqual(wait_for_server.call_args.kwargs["users"], ("root",)) + + def test_a_setup_retry_after_wireguard_needs_no_public_ipv4(self) -> None: + server = self.server() + server.public_ipv4_address = None + provider = Mock(ssh_users=("root",)) + provisioner = ServerProvisioner(server, provider) + + with patch("atlas.metal_server.core.provisioning.wait_for_server", return_value="root") as wait: + provisioner.wait_for_root_ssh() + + self.assertEqual(wait.call_args.kwargs["host"], "fdab:1::1") + + def test_an_unreachable_wireguard_link_is_a_retryable_failure(self) -> None: + provisioner = ServerProvisioner(self.server(), Mock()) + + with ( + patch("atlas.metal_server.core.provisioning.AtlasPeer"), + patch("atlas.metal_server.core.provisioning.wait_for_server", side_effect=TimeoutError), + self.assertRaisesRegex(ProviderOperationError, "through wg0") as raised, + ): + provisioner.wait_for_wireguard_ssh() + + self.assertTrue(raised.exception.is_retryable) + @staticmethod def server() -> SimpleNamespace: server = SimpleNamespace( @@ -111,6 +152,8 @@ def server() -> SimpleNamespace: private_network_mac_address="aa:bb:cc:dd:ee:01", wireguard_ip_address="fdab:1::1", wireguard_public_key="public-key", + ssh_host="fdab:1::1", + settings=SimpleNamespace(), db_set=Mock(), enqueue_disk_sync=Mock(), ) diff --git a/atlas/metal_server/doctype/metal_server/metal_server.py b/atlas/metal_server/doctype/metal_server/metal_server.py index 591724047..a624c85a0 100644 --- a/atlas/metal_server/doctype/metal_server/metal_server.py +++ b/atlas/metal_server/doctype/metal_server/metal_server.py @@ -72,12 +72,19 @@ class MetalServer(Document): @property def ssh_host(self) -> str: - """Return the address an SSH Task connects to.""" + """Return the wg0 address once the host holds the Atlas peer. Setup before that uses the public address.""" + if self.wireguard_public_key and self.wireguard_ip_address: + return self.wireguard_ip_address + if not self.public_ipv4_address: frappe.throw(_("Metal Server {0} has no public IPv4 address.").format(self.name)) return self.public_ipv4_address + def get_ssh_proxy_command(self) -> None: + """Return no proxy. Atlas connects to the host directly.""" + return None + @property def settings(self) -> AtlasSettings: """Return the Atlas settings this server uses.""" @@ -402,6 +409,35 @@ def provision( server.insert(ignore_permissions=True) return server + @staticmethod + def import_from_provider(provider_server_id: str, storage_pool_device: str | None = None) -> MetalServer: + """Insert a Metal Server for a provider server that Atlas did not create, or resume its setup.""" + if not provider_server_id: + frappe.throw(_("Enter the provider server ID.")) + with frappe.db.advisory_lock(f"{frappe.db.cur_db_name}:host-import:{provider_server_id}"): + frappe.db.rollback() # nosemgrep + name = frappe.db.get_value( + "Metal Server", {"provider_server_id": provider_server_id, "status": ["!=", "Deleted"]} + ) + if name: + server: MetalServer = frappe.get_doc("Metal Server", name) + if not server.is_provisioning_completed and not is_job_enqueued(server.setup_job_id): + server._set_storage_pool_device(storage_pool_device) + server.db_set({"provider_metadata": server.provider_metadata, "status": "Pending"}) + server._enqueue_setup_server() + else: + server = frappe.new_doc("Metal Server") + server.provider_server_id = provider_server_id + server._set_storage_pool_device(storage_pool_device) + server.settings.server_provider_controller.import_server(server) + server.architecture = frappe.db.get_value( + "Metal Server Size", server.server_size, "architecture" + ) + server.status = "Pending" + server.insert(ignore_permissions=True) + frappe.db.commit() # nosemgrep + return server + # Internal methods @run_as_admin @@ -451,6 +487,12 @@ def _provider_server_id(self) -> str: frappe.throw(_("Metal Server {0} has no provider server ID.").format(self.name)) return self.provider_server_id + def _set_storage_pool_device(self, storage_pool_device: str | None) -> None: + """Store the storage pool device that an import named in place of the provider device.""" + if storage_pool_device: + metadata = self._provider_metadata(self.provider_metadata) + self.provider_metadata = frappe.as_json({**metadata, "storage_pool_device": storage_pool_device}) + @staticmethod def _provider_metadata(value: str | None) -> dict: """Return provider metadata as an object.""" @@ -481,6 +523,13 @@ def renew_expiring_tls_certificates() -> None: server.enqueue_tls_certificate_renewal() +@frappe.whitelist(methods=["POST"]) +def import_server(provider_server_id: str, storage_pool_device: str | None = None) -> str: + """Add a provider server that was created outside Atlas, and set it up.""" + frappe.only_for("System Manager") + return MetalServer.import_from_provider(provider_server_id.strip(), storage_pool_device or None).name + + @frappe.whitelist(methods=["POST"]) def inspect_host(public_ipv4_address: str, private_ipv4_address: str) -> str: """Queue a read-only inspection of an operator-prepared host.""" diff --git a/atlas/metal_server/doctype/metal_server/metal_server_list.js b/atlas/metal_server/doctype/metal_server/metal_server_list.js index e5345a8a0..43768e9c1 100644 --- a/atlas/metal_server/doctype/metal_server/metal_server_list.js +++ b/atlas/metal_server/doctype/metal_server/metal_server_list.js @@ -394,10 +394,51 @@ async function upgradeMetaldOnSelectedServers(listview) { frappe.set_route("List", "SSH Task"); } +function showImportServerDialog(listview) { + const dialog = new frappe.ui.Dialog({ + title: __("Import Server"), + fields: [ + { + fieldname: "provider_server_id", + fieldtype: "Data", + label: __("Provider Server ID"), + reqd: 1, + description: __("Atlas key must allow root SSH. Atlas adds the private network."), + }, + { + fieldname: "storage_pool_device", + fieldtype: "Data", + label: __("Storage Pool Device"), + description: __( + "Leave empty to use the provider storage device. Atlas destroys all data on it." + ), + }, + ], + primary_action_label: __("Import"), + primary_action(values) { + frappe + .call({ method: `${HOST_REGISTRATION_METHOD}.import_server`, args: values }) + .then(({ message }) => { + dialog.hide(); + listview.refresh(); + frappe.set_route("Form", "Metal Server", message); + }); + }, + }); + dialog.show(); +} + frappe.listview_settings["Metal Server"] = { add_fields: ["title"], onload(listview) { if (!frappe.user.has_role("System Manager")) return; + frappe.db.get_single_value("Atlas Settings", "server_provider").then((provider) => { + if (provider !== "Generic") { + listview.page.add_inner_button(__("Import Server"), () => + showImportServerDialog(listview) + ); + } + }); listview.page.add_actions_menu_item(__("Upgrade Metald"), () => frappe.confirm( __("Upgrade Metald on {0} selected servers?", [ diff --git a/atlas/metal_server/doctype/metal_server/test_metal_server.py b/atlas/metal_server/doctype/metal_server/test_metal_server.py index 6f87ac3df..89adb12a6 100644 --- a/atlas/metal_server/doctype/metal_server/test_metal_server.py +++ b/atlas/metal_server/doctype/metal_server/test_metal_server.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from contextlib import nullcontext from datetime import datetime from types import MethodType, SimpleNamespace from unittest.mock import Mock, patch @@ -8,7 +9,7 @@ import frappe from frappe.tests import UnitTestCase -from atlas.atlas.core.server_providers.base import ProviderServer, ServerPowerAction +from atlas.atlas.core.server_providers.base import ProviderServer, ServerPowerAction, ServerProvider from atlas.atlas.core.tls.metal import CERTIFICATE_RENEWAL_WINDOW_DAYS from atlas.metal_server.doctype.metal_server.metal_server import ( MetalServer, @@ -514,9 +515,21 @@ def test_install_metald_worker_passes_the_pool_device(self) -> None: ssh_runner.return_value.run_script.return_value = tls_result MetalServer._install_metald(server) - storage_arguments, arguments = (call.kwargs for call in create_for_script_file.call_args_list) + storage_arguments, arguments, firewall_arguments = ( + call.kwargs for call in create_for_script_file.call_args_list + ) self.assertEqual(storage_arguments["script_path"], "install-metal-storage.sh") self.assertEqual(storage_arguments["environment"], {"STORAGE_POOL_DEVICE": "/dev/md2"}) + self.assertEqual(firewall_arguments["script_path"], "install-host-firewall.sh") + self.assertEqual( + firewall_arguments["environment"], + { + "ATLAS_WIREGUARD_ADDRESS": "fdaa:1::ffff:ffff:ffff:ffff", + "MESH_UPLINK_INTERFACE": "eno1.1878", + "PRIVATE_NETWORK_CIDR": "10.0.0.0/20", + "WIREGUARD_LISTEN_PORT": 51820, + }, + ) self.assertEqual(arguments["script_path"], "install-metald.sh") self.assertEqual( arguments["environment"], @@ -525,10 +538,12 @@ def test_install_metald_worker_passes_the_pool_device(self) -> None: "METALD_SHA256": "metald-binary-sha256", "WG_MESH_DOWNLOAD_URL": "https://atlas.test/files/atlas-wg-mesh-linux-amd64", "WG_MESH_SHA256": "wg-mesh-binary-sha256", - "LISTEN_ADDRESS": "10.0.0.7:9000", + "LISTEN_ADDRESS": "[fdab:1::7]:9000", "ATLAS_COMMON_NAME": "atlas.example.test", "COORDINATION_LISTEN_ADDRESS": "[fdab:1::7]:9001", "MESH_UPLINK_INTERFACE": "eno1.1878", + "PRIVATE_NETWORK_CIDR": "10.0.0.0/20", + "ATLAS_MESH_ADDRESS": "fdaa:1::ffff:ffff:ffff:ffff", }, ) ssh_runner.return_value.run_script.assert_called_once_with( @@ -541,49 +556,6 @@ def test_install_metald_worker_passes_the_pool_device(self) -> None: timeout_seconds=1200, ) - def test_install_metald_listens_on_the_address_the_provider_chooses(self) -> None: - server = self._server(status="Running") - server.settings.metald_binary_x86_64_file = "metald-file" - server.settings.server_provider_controller.metald_listen_address = Mock(return_value="203.0.113.7") - server.wireguard_ip_address = "fdab:1::7" - task = SimpleNamespace(result=SimpleNamespace(is_success=True)) - tls_result = SimpleNamespace(is_success=True) - - with ( - patch( - "atlas.metal_server.core.host_installation.get_download_url", - return_value="https://atlas.test/files/metald-linux-amd64", - ), - patch( - "atlas.metal_server.core.host_installation.SSHTask.create_for_script_file", - return_value=task, - ) as create_for_script_file, - patch( - "atlas.metal_server.core.host_installation.ensure_server_certificate", - return_value=("ca", "certificate", "private-key"), - ), - patch("atlas.metal_server.core.host_installation.SSHRunner") as ssh_runner, - ): - ssh_runner.return_value.run_script.return_value = tls_result - MetalServer._install_metald(server) - - self.assertEqual( - create_for_script_file.call_args.kwargs["environment"]["LISTEN_ADDRESS"], "203.0.113.7:9000" - ) - - def test_install_metald_rejects_an_address_that_is_not_ipv4(self) -> None: - server = self._server(status="Running") - server.settings.metald_binary_x86_64_file = "metald-file" - server.settings.server_provider_controller.metald_listen_address = Mock(return_value="0.0.0.0/0") - - with ( - patch("atlas.metal_server.core.host_installation.HostInstallation.install_storage"), - patch("atlas.metal_server.core.host_installation.HostInstallation.install_tls_credentials"), - patch("atlas.metal_server.core.host_installation.frappe.throw", side_effect=ValueError), - ): - with self.assertRaises(ValueError): - MetalServer._install_metald(server) - def test_tls_renewal_waits_for_a_running_metald_job(self) -> None: server = self._server(status="Running") @@ -758,6 +730,87 @@ def test_install_metald_worker_needs_a_private_network_interface(self) -> None: create_for_script_file.assert_not_called() + def test_import_continues_the_setup_of_a_known_server(self) -> None: + """Importing the same provider server again must not create a second record.""" + server = self._server(status="Failed") + server._enqueue_setup_server = Mock() + server._set_storage_pool_device = MethodType(MetalServer._set_storage_pool_device, server) + + with ( + patch( + "atlas.metal_server.doctype.metal_server.metal_server.frappe.db.advisory_lock", + return_value=nullcontext(), + ), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.db.rollback"), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.db.commit") as commit, + patch( + "atlas.metal_server.doctype.metal_server.metal_server.frappe.db.get_value", + return_value=SERVER_NAME, + ), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.get_doc", return_value=server), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.new_doc") as new_doc, + patch("atlas.metal_server.doctype.metal_server.metal_server.is_job_enqueued", return_value=False), + ): + MetalServer.import_from_provider("server-id", "/root/disks/atlas.img") + + new_doc.assert_not_called() + commit.assert_called_once_with() + server._enqueue_setup_server.assert_called_once_with() + self.assertEqual(server.status, "Pending") + self.assertEqual(json.loads(server.provider_metadata)["storage_pool_device"], "/root/disks/atlas.img") + + def test_import_inserts_a_pending_server_from_the_provider(self) -> None: + server = self._server(status="Pending") + server.provider_server_id = None + server.insert = Mock() + server._set_storage_pool_device = MethodType(MetalServer._set_storage_pool_device, server) + provider = server.settings.server_provider_controller + provider.import_server = Mock( + side_effect=lambda imported: setattr(imported, "server_size", "EM-A116X-SSD") + ) + + with ( + patch( + "atlas.metal_server.doctype.metal_server.metal_server.frappe.db.advisory_lock", + return_value=nullcontext(), + ), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.db.rollback"), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.db.commit") as commit, + patch( + "atlas.metal_server.doctype.metal_server.metal_server.frappe.db.get_value", + side_effect=[None, "amd64"], + ), + patch("atlas.metal_server.doctype.metal_server.metal_server.frappe.new_doc", return_value=server), + ): + MetalServer.import_from_provider("server-id") + + provider.import_server.assert_called_once_with(server) + self.assertEqual( + (server.provider_server_id, server.architecture, server.status), ("server-id", "amd64", "Pending") + ) + server.insert.assert_called_once_with(ignore_permissions=True) + commit.assert_called_once_with() + + def test_an_imported_storage_device_replaces_the_provider_device(self) -> None: + server = self._server(status="Running") + server.provider_metadata = json.dumps({"storage_pool_device": "/root/disks/atlas.img"}) + + provider = SimpleNamespace(storage_pool_device=Mock(return_value="/dev/md2")) + self.assertEqual(ServerProvider.get_storage_pool_device(provider, server), "/root/disks/atlas.img") + + server.provider_metadata = "{}" + self.assertEqual(ServerProvider.get_storage_pool_device(provider, server), "/dev/md2") + + def test_ssh_uses_wg0_once_the_host_has_a_wireguard_key(self) -> None: + server = self._server(status="Running") + server.wireguard_ip_address = "fdab:1::7" + + server.wireguard_public_key = None + self.assertEqual(MetalServer.ssh_host.fget(server), "203.0.113.7") + + server.wireguard_public_key = "host-key" + self.assertEqual(MetalServer.ssh_host.fget(server), "fdab:1::7") + def test_get_wireguard_ip_address_uses_the_server_uuid(self) -> None: server = self._server(status="Running") @@ -813,7 +866,7 @@ def test_configure_wireguard_job_stores_the_address_and_public_key(self) -> None server = self._server(status="Running") output = ( "==> packages\n==> interface (wg0)\n" - "===PUBLIC_KEY_START===\nSGVsbG9XaXJlR3VhcmRQdWJsaWNLZXlIZXJlPQ=\n===PUBLIC_KEY_END===\n" + "===PUBLIC_KEY_START===\nC9VOcyTb+m1vKqzRZbDsbQ55e1OIzEwSj+ttwbjMv1w=\n===PUBLIC_KEY_END===\n" ) task = SimpleNamespace(result=SimpleNamespace(output=output, is_success=True)) @@ -830,11 +883,51 @@ def test_configure_wireguard_job_stores_the_address_and_public_key(self) -> None "WIREGUARD_ADDRESS": SERVER_MESH_ADDRESS, "WIREGUARD_LISTEN_PORT": 51820, "MESH_UPLINK_INTERFACE": "eno1.1878", + "ATLAS_WIREGUARD_ADDRESS": "fdaa:1::ffff:ffff:ffff:ffff", + "ATLAS_WIREGUARD_PUBLIC_KEY": "atlas-key", }, ) self.assertFalse(arguments["run_in_background"]) server.db_set.assert_any_call("wireguard_ip_address", SERVER_MESH_ADDRESS) - server.db_set.assert_called_with("wireguard_public_key", "SGVsbG9XaXJlR3VhcmRQdWJsaWNLZXlIZXJlPQ=") + server.db_set.assert_called_with( + "wireguard_public_key", "C9VOcyTb+m1vKqzRZbDsbQ55e1OIzEwSj+ttwbjMv1w=" + ) + + def test_configure_wireguard_ignores_a_progress_line_between_the_markers(self) -> None: + server = self._server(status="Running") + output = ( + "===PUBLIC_KEY_START===\nC9VOcyTb+m1vKqzRZbDsbQ55e1OIzEwSj+ttwbjMv1w=\n" + "==> Atlas peer (fdaa:1::ffff:ffff:ffff:ffff)\n===PUBLIC_KEY_END===\n" + ) + task = SimpleNamespace(result=SimpleNamespace(output=output, is_success=True)) + + with ( + patch( + "atlas.metal_server.core.host_installation.SSHTask.create_for_script_file", + return_value=task, + ), + ): + MetalServer._configure_wireguard(server) + + server.db_set.assert_called_with( + "wireguard_public_key", "C9VOcyTb+m1vKqzRZbDsbQ55e1OIzEwSj+ttwbjMv1w=" + ) + + def test_configure_wireguard_needs_the_atlas_identity(self) -> None: + """Only configure-atlas-wireguard creates the key, so two first provisions cannot make two keys.""" + server = self._server(status="Running") + + server.settings.wireguard_public_key = None + + with ( + patch( + "atlas.metal_server.core.host_installation.SSHTask.create_for_script_file" + ) as create_for_script_file, + self.assertRaisesRegex(frappe.ValidationError, "configure-atlas-wireguard"), + ): + MetalServer._configure_wireguard(server) + + create_for_script_file.assert_not_called() def test_configure_wireguard_job_needs_the_mesh_uplink(self) -> None: server = self._server(status="Running") @@ -1057,7 +1150,7 @@ def _server(*, status: str) -> SimpleNamespace: delete_server=Mock(), set_power_state=Mock(), storage_pool_device=Mock(return_value="/dev/md2"), - metald_listen_address=Mock(return_value="10.0.0.7"), + get_storage_pool_device=Mock(return_value="/dev/md2"), ), metald_binary_x86_64_file=None, metald_binary_hash="metald-binary-sha256", @@ -1066,7 +1159,9 @@ def _server(*, status: str) -> SimpleNamespace: wildcard_domain="example.test", region_id=1, private_network_mtu=1500, - use_public_ip_for_metald=False, + private_network_cidr="10.0.0.0/20", + wireguard_ip_address="fdaa:1::ffff:ffff:ffff:ffff", + wireguard_public_key="atlas-key", ), set=Mock(), save=Mock(), diff --git a/atlas/scripts/configure-wireguard.sh b/atlas/scripts/configure-wireguard.sh index 59dbfcd24..108dd8907 100755 --- a/atlas/scripts/configure-wireguard.sh +++ b/atlas/scripts/configure-wireguard.sh @@ -4,6 +4,8 @@ set -eu : "${WIREGUARD_ADDRESS:?WIREGUARD_ADDRESS is required}" : "${MESH_UPLINK_INTERFACE:?MESH_UPLINK_INTERFACE is required}" +: "${ATLAS_WIREGUARD_ADDRESS:?ATLAS_WIREGUARD_ADDRESS is required}" +: "${ATLAS_WIREGUARD_PUBLIC_KEY:?ATLAS_WIREGUARD_PUBLIC_KEY is required}" interface=${WIREGUARD_INTERFACE:-wg0} listen_port=${WIREGUARD_LISTEN_PORT:-51820} @@ -25,6 +27,14 @@ fdab:*) ;; ;; esac +case "$ATLAS_WIREGUARD_ADDRESS" in +fdaa:*) ;; +*) + echo "ATLAS_WIREGUARD_ADDRESS must be inside fdaa::/16, got $ATLAS_WIREGUARD_ADDRESS" >&2 + exit 1 + ;; +esac + step() { echo "==> $*" >&2; } # The tunnel crosses the mesh uplink. The overhead is one IPv4 header, one UDP @@ -55,28 +65,46 @@ if [ ! -f "$private_key_file" ]; then fi step "config ($config_file)" -# The region prefix makes every peer on-link. The daemon adds peers, and +# The region prefix makes every peer on-link. The daemon adds host peers, and # `wg set` installs no route of its own. -config="[Interface] +interface_config="[Interface] Address = $WIREGUARD_ADDRESS/32 ListenPort = $listen_port MTU = $wireguard_mtu PostUp = wg set %i private-key $private_key_file" +config="$interface_config + +[Peer] +PublicKey = $ATLAS_WIREGUARD_PUBLIC_KEY +AllowedIPs = $ATLAS_WIREGUARD_ADDRESS/128" + # A reused host keeps the config of its earlier registration, so rewrite a stale one. -is_config_changed=false -if [ ! -f "$config_file" ] || [ "$(cat "$config_file")" != "$config" ]; then +# A wg0 restart drops the host peers until the next sync. +previous_config=$(cat "$config_file" 2>/dev/null || true) +previous_atlas_public_key=$(printf '%s\n' "$previous_config" | sed -n 's/^PublicKey = //p') +is_interface_changed=false +if [ "$(printf '%s\n' "$previous_config" | sed '/^$/,$d')" != "$interface_config" ]; then + is_interface_changed=true +fi +if [ "$previous_config" != "$config" ]; then (umask 077 && printf '%s\n' "$config" > "$config_file") - is_config_changed=true fi step "interface ($interface)" systemctl enable --now "wg-quick@$interface" -if [ "$is_config_changed" = true ]; then +if [ "$is_interface_changed" = true ]; then systemctl restart "wg-quick@$interface" fi systemctl is-active "wg-quick@$interface" >/dev/null +step "Atlas peer ($ATLAS_WIREGUARD_ADDRESS)" +if [ -n "$previous_atlas_public_key" ] && [ "$previous_atlas_public_key" != "$ATLAS_WIREGUARD_PUBLIC_KEY" ]; then + wg set "$interface" peer "$previous_atlas_public_key" remove +fi +wg set "$interface" peer "$ATLAS_WIREGUARD_PUBLIC_KEY" allowed-ips "$ATLAS_WIREGUARD_ADDRESS/128" +ip -6 route replace "$ATLAS_WIREGUARD_ADDRESS/128" dev "$interface" + # !!! DON'T CHANGE THE FORMAT OF BELOW OUTPUT !!! diff --git a/atlas/scripts/install-host-firewall.sh b/atlas/scripts/install-host-firewall.sh new file mode 100755 index 000000000..7c18f3194 --- /dev/null +++ b/atlas/scripts/install-host-firewall.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Install the host input firewall. It covers traffic to the host itself. Metal filters forwarded guest traffic. + +set -eu + +: "${ATLAS_WIREGUARD_ADDRESS:?ATLAS_WIREGUARD_ADDRESS is required}" +: "${MESH_UPLINK_INTERFACE:?MESH_UPLINK_INTERFACE is required}" +: "${PRIVATE_NETWORK_CIDR:?PRIVATE_NETWORK_CIDR is required}" + +wireguard_interface=${WIREGUARD_INTERFACE:-wg0} +wireguard_port=${WIREGUARD_LISTEN_PORT:-51820} +rules_file=/etc/atlas/host-firewall.nft + +if [ "$(id -u)" -ne 0 ]; then + echo "install-host-firewall must run as root" >&2 + exit 1 +fi + +wireguard_rule="iifname \"$MESH_UPLINK_INTERFACE\" ip saddr $PRIVATE_NETWORK_CIDR udp dport $wireguard_port accept" + +if ! command -v nft >/dev/null; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq nftables +fi + +install -d -m 0755 /etc/atlas +cat > "$rules_file.staged" < /etc/systemd/system/atlas-host-firewall.service < host firewall active" diff --git a/atlas/scripts/install-metald.sh b/atlas/scripts/install-metald.sh index 4f1c17a38..49c69d28e 100755 --- a/atlas/scripts/install-metald.sh +++ b/atlas/scripts/install-metald.sh @@ -6,10 +6,12 @@ set -eu : "${METALD_DOWNLOAD_URL:?METALD_DOWNLOAD_URL is required}" : "${METALD_SHA256:?METALD_SHA256 is required}" : "${MESH_UPLINK_INTERFACE:?MESH_UPLINK_INTERFACE is required}" +: "${PRIVATE_NETWORK_CIDR:?PRIVATE_NETWORK_CIDR is required}" : "${WG_MESH_DOWNLOAD_URL:?WG_MESH_DOWNLOAD_URL is required}" : "${WG_MESH_SHA256:?WG_MESH_SHA256 is required}" : "${COORDINATION_LISTEN_ADDRESS:?COORDINATION_LISTEN_ADDRESS is required}" : "${ATLAS_COMMON_NAME:?ATLAS_COMMON_NAME is required}" +: "${ATLAS_MESH_ADDRESS:?ATLAS_MESH_ADDRESS is required}" storage_pool_name=${STORAGE_POOL_NAME:-metal} firecracker_version=${FIRECRACKER_VERSION:-v1.16.1} @@ -54,10 +56,10 @@ service_is_stable() { step "install required packages" -if ! command -v zpool >/dev/null || ! command -v curl >/dev/null || ! command -v iptables >/dev/null || ! command -v openssl >/dev/null; then +if ! command -v zpool >/dev/null || ! command -v curl >/dev/null || ! command -v iptables >/dev/null || ! command -v openssl >/dev/null || ! command -v nc >/dev/null; then export DEBIAN_FRONTEND=noninteractive apt update -qq - apt install -y -qq curl iptables openssl tar zfsutils-linux + apt install -y -qq curl iptables netcat-openbsd openssl tar zfsutils-linux else skip "packages" fi @@ -147,6 +149,7 @@ interface = "$wireguard_interface" [wg_mesh] binary_path = "$mesh_binary_path" uplink = "$MESH_UPLINK_INTERFACE" +controller_address = "$ATLAS_MESH_ADDRESS" EOF } @@ -181,6 +184,11 @@ if [ -f "$config_file" ]; then sed -i "/^auth_token_hash[[:space:]]*=/d" "$config_file" if grep -q '^\[wg_mesh\]' "$config_file"; then sed -i "s|^uplink = .*|uplink = \"$MESH_UPLINK_INTERFACE\"|" "$config_file" + if grep -q '^controller_address[[:space:]]*=' "$config_file"; then + sed -i "s|^controller_address[[:space:]]*=.*|controller_address = \"$ATLAS_MESH_ADDRESS\"|" "$config_file" + else + sed -i "/^uplink = /a controller_address = \"$ATLAS_MESH_ADDRESS\"" "$config_file" + fi sed -i "s|^binary_path = \"/usr/local/bin/atlas-wg-mesh\"|binary_path = \"$mesh_binary_path\"|" "$config_file" else mesh_sections @@ -213,18 +221,21 @@ fi step "network setup" install -d -m 0755 /usr/local/lib/metal -cat > /usr/local/lib/metal/network-setup <<'EOF' +cat > /usr/local/lib/metal/network-setup <&2 exit 1 } -iptables -t nat -C POSTROUTING -s 10.0.0.0/8 -o "$uplink" -j MASQUERADE 2>/dev/null || - iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -o "$uplink" -j MASQUERADE +iptables -C FORWARD -i vh+ -d "$PRIVATE_NETWORK_CIDR" -j DROP 2>/dev/null || + iptables -A FORWARD -i vh+ -d "$PRIVATE_NETWORK_CIDR" -j DROP + +iptables -t nat -C POSTROUTING -s 10.0.0.0/8 -o "\$uplink" -j MASQUERADE 2>/dev/null || + iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -o "\$uplink" -j MASQUERADE EOF chmod 0755 /usr/local/lib/metal/network-setup @@ -233,8 +244,8 @@ step "systemd units" cat > /etc/systemd/system/metal.service </dev/null || echo "table inet atlas_host is absent" diff --git a/atlas/service/SPEC.md b/atlas/service/SPEC.md index e6baa233e..d3fdfa316 100644 --- a/atlas/service/SPEC.md +++ b/atlas/service/SPEC.md @@ -16,7 +16,8 @@ Behavior: [Service VMs](../../docs/region/service-vms.md). This module runs Atla ## Shared rules -- Each VM is created through `VirtualMachineService` as a privileged tenant-0 VM. It needs a reserved tenant-0 IPv4 allocation. +- Each VM is created through `VirtualMachineService` as a privileged tenant-0 VM. Only a Proxy VM needs a reserved tenant-0 IPv4 allocation. Its guest firewall, from `get_proxy_firewall`, admits only TCP 80 and 443 from public addresses. +- Atlas reaches service SSH through the VM host. See [Atlas access to hosts](../../docs/region/host-access.md#ssh). - A job requeues `Pending` records every minute. - A failure sets `Failed` with the phase and message. Nothing replaces the VM automatically. - A site file lock guards each record. Code reads the record again under the lock. @@ -26,6 +27,7 @@ Behavior: [Service VMs](../../docs/region/service-vms.md). This module runs Atla ## Proxy Server - Atlas sends the new peer list to active nodes before it publishes a node in regional DNS. +- The apply command maps each peer name to its mesh address in `/etc/hosts`, so replication stays on the mesh with the same name and certificate. - A job pushes a changed configuration digest every minute. See the [HTTP proxy specification](../../services/http-proxy/SPEC.md). diff --git a/atlas/service/core/cargo/provisioning.py b/atlas/service/core/cargo/provisioning.py index e978ef6d9..76a130fd8 100644 --- a/atlas/service/core/cargo/provisioning.py +++ b/atlas/service/core/cargo/provisioning.py @@ -13,6 +13,7 @@ import requests from frappe import _ +from atlas.atlas.core.artifacts import get_internal_base_url from atlas.atlas.core.ssh import wait_for_server from atlas.atlas.doctype.ssh_task.ssh_task import SSHTask from atlas.auth.issuer import issue_token @@ -112,12 +113,13 @@ def proxy_url(self) -> str: return f"https://proxy.{self.settings.wildcard_domain}" def wait_for_ssh(self) -> None: - """Wait for root SSH on the public IPv4 address.""" + """Wait for root SSH through the guest host.""" wait_for_server( host=self.virtual_machine.ssh_host, users=("root",), timeout_seconds=SSH_TIMEOUT_SECONDS, poll_interval_seconds=SSH_POLL_INTERVAL_SECONDS, + proxy_command=self.virtual_machine.get_ssh_proxy_command(), ) def install_cargo(self) -> None: @@ -157,9 +159,7 @@ def install_environment(self) -> dict[str, str | int]: constraints={"site": {"suffix": "-svc"}}, lifetime=TOKEN_LIFETIME, ) - atlas_url = (frappe.conf.atlas_base_url or frappe.utils.get_url(allow_header_override=False)).rstrip( - "/" - ) + atlas_url = get_internal_base_url().rstrip("/") return { "PILOT_ADMIN_PASSWORD": generate_installer_password(), diff --git a/atlas/service/core/cargo/test_provisioning.py b/atlas/service/core/cargo/test_provisioning.py index 3ea066e85..4810e6625 100644 --- a/atlas/service/core/cargo/test_provisioning.py +++ b/atlas/service/core/cargo/test_provisioning.py @@ -114,8 +114,8 @@ def test_a_failed_step_records_its_phase(self) -> None: self.assertEqual(server.status, "Failed") self.assertEqual(server.failure_message, "proxy-routes: proxy unavailable") - def test_ssh_wait_uses_the_virtual_machine_public_address(self) -> None: - virtual_machine = SimpleNamespace(ssh_host="203.0.113.9") + def test_ssh_wait_goes_through_the_guest_host(self) -> None: + virtual_machine = SimpleNamespace(ssh_host="vm-00001", get_ssh_proxy_command=lambda: "ssh host nc") provisioner = CargoServerProvisioner(cargo_server()) with ( patch.object( @@ -126,10 +126,11 @@ def test_ssh_wait_uses_the_virtual_machine_public_address(self) -> None: provisioner.wait_for_ssh() wait_for_server.assert_called_once_with( - host="203.0.113.9", + host="vm-00001", users=("root",), timeout_seconds=600, poll_interval_seconds=5, + proxy_command="ssh host nc", ) def test_installation_uses_a_visible_ssh_task(self) -> None: diff --git a/atlas/service/core/ipv6_router/provisioning.py b/atlas/service/core/ipv6_router/provisioning.py index 3bba89b75..142802a9e 100644 --- a/atlas/service/core/ipv6_router/provisioning.py +++ b/atlas/service/core/ipv6_router/provisioning.py @@ -73,19 +73,12 @@ def steps(self) -> tuple[tuple[str, Callable[[], None]], ...]: @property def is_virtual_machine_ready(self) -> bool: - """Report whether the VM left the draft state and Metal holds its public IPv4 address. - - The network step replaces the complete Metal network. It must not overlap the IPv4 reconcile job. - """ + """Report whether the VM left the draft state and Metal holds it.""" if not self.router.virtual_machine: return False virtual_machine = self.virtual_machine - if virtual_machine.is_draft: - return False - - information = virtual_machine.get_metal_vm_info() - return bool(information and information.desired.network.public_ipv4) + return not virtual_machine.is_draft and bool(virtual_machine.get_metal_vm_info()) @property def virtual_machine(self) -> VirtualMachine: @@ -119,12 +112,13 @@ def configure_network(self) -> None: service.update_network({"public_ipv6": pool.prefix, "routes": service.get_routes_with(route)}) def wait_for_ssh(self) -> None: - """Wait for root SSH on the public IPv4 address.""" + """Wait for root SSH through the guest host.""" wait_for_server( host=self.virtual_machine.ssh_host, users=("root",), timeout_seconds=SSH_TIMEOUT_SECONDS, poll_interval_seconds=SSH_POLL_INTERVAL_SECONDS, + proxy_command=self.virtual_machine.get_ssh_proxy_command(), ) def install_router(self) -> None: diff --git a/atlas/service/core/ipv6_router/test_provisioning.py b/atlas/service/core/ipv6_router/test_provisioning.py index 86705ffac..0769f0b53 100644 --- a/atlas/service/core/ipv6_router/test_provisioning.py +++ b/atlas/service/core/ipv6_router/test_provisioning.py @@ -50,12 +50,8 @@ def is_ready(self, public_ipv4: str) -> bool: with patch.object(provisioning.frappe, "get_doc", return_value=machine): return IPv6RouterServerProvisioner(router()).is_virtual_machine_ready - # The network step would overwrite the address that the IPv4 reconcile job sends to Metal. - def test_the_router_waits_until_metal_holds_the_public_ipv4(self) -> None: - self.assertFalse(self.is_ready("")) - - def test_the_router_is_ready_with_the_public_ipv4(self) -> None: - self.assertTrue(self.is_ready("151.115.112.94")) + def test_the_router_needs_no_public_ipv4(self) -> None: + self.assertTrue(self.is_ready("")) class TestRouterNetwork(UnitTestCase): diff --git a/atlas/service/core/proxy/configuration.py b/atlas/service/core/proxy/configuration.py index f9b309454..242dfee40 100644 --- a/atlas/service/core/proxy/configuration.py +++ b/atlas/service/core/proxy/configuration.py @@ -2,6 +2,7 @@ import hashlib import json +import shlex from datetime import UTC, timedelta from functools import cached_property from string import Template @@ -13,7 +14,8 @@ from frappe import _ from frappe.utils import get_datetime, get_system_timezone -from atlas.atlas.core.mesh_address import get_region_mesh_address_prefix +from atlas.atlas.core.mesh_address import get_region_mesh_address_prefix, get_virtual_machine_mesh_address +from atlas.vm.doctype.virtual_machine.virtual_machine import PRIVILEGED_TENANT_ID if TYPE_CHECKING: from atlas.atlas.doctype.atlas_settings.atlas_settings import AtlasSettings @@ -70,8 +72,12 @@ mv -f $temporary_path $config_path""" ) +PEER_HOSTS_MARKER = "# atlas-proxy-peer" + APPLY_COMMAND_TEMPLATE = Template( """set -eu +sed -i '/ $peer_hosts_marker$$/d' /etc/hosts +printf '%s' $peer_hosts >> /etc/hosts $apply_command systemctl enable --now $socket_unit systemctl enable $daemon_unit @@ -187,6 +193,12 @@ def peers(self) -> list[dict[str, str]]: { "node_id": proxy_server.name, "address": f"https://{proxy_server.get_domain()}", + "mesh_address": get_virtual_machine_mesh_address( + frappe._dict(name=proxy_server.virtual_machine, tenant_id=PRIVILEGED_TENANT_ID), + self.settings.region_id, + ) + if proxy_server.virtual_machine + else "", } for proxy_server in sorted(proxy_servers, key=lambda item: item.name) ] @@ -231,8 +243,17 @@ def get_write_command(self) -> str: def get_apply_command(self) -> str: """Return the command that applies the configuration.""" + peer_hosts = "".join( + f"{peer['mesh_address']} {peer['address'].removeprefix('https://')} {PEER_HOSTS_MARKER}\n" + for peer in self.peers + if peer["mesh_address"] + ) return APPLY_COMMAND_TEMPLATE.substitute( - apply_command=APPLY_COMMAND, socket_unit=DAEMON_SOCKET_UNIT, daemon_unit=DAEMON_UNIT + peer_hosts_marker=PEER_HOSTS_MARKER, + peer_hosts=shlex.quote(peer_hosts), + apply_command=APPLY_COMMAND, + socket_unit=DAEMON_SOCKET_UNIT, + daemon_unit=DAEMON_UNIT, ) diff --git a/atlas/service/core/proxy/provisioning.py b/atlas/service/core/proxy/provisioning.py index 7d7fd41c2..b03971203 100644 --- a/atlas/service/core/proxy/provisioning.py +++ b/atlas/service/core/proxy/provisioning.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from atlas.service.doctype.proxy_server.proxy_server import ProxyServer + from atlas.vm.doctype.virtual_machine.virtual_machine import VirtualMachine INSTALL_TIMEOUT_SECONDS = 1_800 CONFIGURE_TIMEOUT_SECONDS = 300 @@ -182,12 +183,14 @@ def wait_for_control_readiness(self, save: bool = True) -> None: frappe.throw(_("Proxy Server {0} did not become ready.").format(self.proxy_server.name)) def wait_for_ssh(self, save: bool = True) -> None: - """Wait until the guest answers as root on its public address.""" + """Wait until the guest answers as root through its host.""" + virtual_machine = self.virtual_machine wait_for_server( - host=self.ssh_host, + host=virtual_machine.ssh_host, users=("root",), timeout_seconds=SSH_TIMEOUT_SECONDS, poll_interval_seconds=SSH_POLL_INTERVAL_SECONDS, + proxy_command=virtual_machine.get_ssh_proxy_command(), ) self.save_progress(save) @@ -198,9 +201,10 @@ def push_configuration(self, save: bool = True) -> None: self.save_progress(save) return - write = SSHRunner(self.ssh_host).run_command( - configuration.get_write_command(), timeout_seconds=CONFIGURE_TIMEOUT_SECONDS - ) + virtual_machine = self.virtual_machine + write = SSHRunner( + virtual_machine.ssh_host, proxy_command=virtual_machine.get_ssh_proxy_command() + ).run_command(configuration.get_write_command(), timeout_seconds=CONFIGURE_TIMEOUT_SECONDS) if not write.is_success: frappe.throw( _("The proxy did not accept its configuration file: {0}").format(write.output.strip()) @@ -244,10 +248,9 @@ def install_package(self, save: bool = True) -> None: self.save_progress(save) @property - def ssh_host(self) -> str: - """Return the address the controller connects to.""" - virtual_machine = frappe.get_doc("Virtual Machine", self.proxy_server.virtual_machine) - return virtual_machine.ssh_host + def virtual_machine(self) -> VirtualMachine: + """Return the proxy guest.""" + return frappe.get_doc("Virtual Machine", self.proxy_server.virtual_machine) def save_progress(self, save: bool = True) -> None: """Store the current proxy state.""" diff --git a/atlas/service/core/proxy/test_configuration.py b/atlas/service/core/proxy/test_configuration.py index 5a662ec1f..be46de6a0 100644 --- a/atlas/service/core/proxy/test_configuration.py +++ b/atlas/service/core/proxy/test_configuration.py @@ -51,6 +51,7 @@ def get_password(self, fieldname: str, raise_exception: bool = True) -> str | No class _FakeProxyServer: name = "proxy-001" + virtual_machine = None def __init__(self, password: str = "a-control-password") -> None: self.password = password @@ -189,6 +190,17 @@ def test_the_apply_command_starts_the_daemon(self) -> None: ) self.assertLess(command.index(APPLY_COMMAND), command.index(f"enable {DAEMON_UNIT}")) + def test_the_apply_command_points_peer_names_at_the_mesh(self) -> None: + """Replication must stay on the mesh, so each peer name resolves to its mesh address.""" + configuration = _build() + configuration.proxy_server.virtual_machine = "vm-0000082" + + with patch("atlas.atlas.core.mesh_address.frappe.get_single", return_value=_FakeSettings()): + command = configuration.get_apply_command() + + self.assertIn("sed -i '/ # atlas-proxy-peer$/d' /etc/hosts", command) + self.assertIn("fdaa:1::52 proxy-001.par-1.example.com # atlas-proxy-peer", command) + class TestPushToActiveProxies(UnitTestCase): def test_only_active_proxies_are_queued(self) -> None: diff --git a/atlas/service/core/proxy/test_provisioning.py b/atlas/service/core/proxy/test_provisioning.py index 67625e84b..0d2e2575d 100644 --- a/atlas/service/core/proxy/test_provisioning.py +++ b/atlas/service/core/proxy/test_provisioning.py @@ -151,7 +151,13 @@ def test_a_rejected_configuration_file_fails_loudly(self) -> None: patch.object(provisioning, "ProxyConfiguration") as configuration, patch.object(provisioning, "SSHRunner") as ssh_runner, patch.object(provisioning.SSHTask, "create_for_command") as create_task, - patch.object(provisioner.__class__, "ssh_host", new=property(lambda self: "203.0.113.9")), + patch.object( + provisioner.__class__, + "virtual_machine", + new=property( + lambda self: SimpleNamespace(ssh_host="vm-00001", get_ssh_proxy_command=lambda: "proxy") + ), + ), ): configuration.return_value.digest = "digest-2" ssh_runner.return_value.run_command.return_value = SSHResult("no space left", 1) @@ -170,7 +176,13 @@ def test_a_failed_apply_step_names_its_ssh_task(self) -> None: patch.object(provisioning, "ProxyConfiguration") as configuration, patch.object(provisioning, "SSHRunner") as ssh_runner, patch.object(provisioning.SSHTask, "create_for_command") as create_task, - patch.object(provisioner.__class__, "ssh_host", new=property(lambda self: "203.0.113.9")), + patch.object( + provisioner.__class__, + "virtual_machine", + new=property( + lambda self: SimpleNamespace(ssh_host="vm-00001", get_ssh_proxy_command=lambda: "proxy") + ), + ), ): configuration.return_value.digest = "digest-3" ssh_runner.return_value.run_command.return_value = SSHResult("", 0) diff --git a/atlas/service/doctype/cargo_server/cargo_server.js b/atlas/service/doctype/cargo_server/cargo_server.js index 28e9177b5..ab68cddbf 100644 --- a/atlas/service/doctype/cargo_server/cargo_server.js +++ b/atlas/service/doctype/cargo_server/cargo_server.js @@ -46,20 +46,6 @@ function showProvisionDialog(frm) { reqd: 1, filters: { enabled: 1, status: "Available", image_type: "system" }, }, - { fieldtype: "Column Break" }, - { - fieldname: "public_ipv4", - fieldtype: "Link", - label: __("Public IPv4"), - options: "Public IP Allocation", - reqd: 1, - filters: { - status: "Reserved", - version: "4", - tenant_id: 0, - virtual_machine: ["is", "not set"], - }, - }, { fieldtype: "Section Break", label: __("Cargo Server Resources") }, intField("cpu_millicores", __("CPU (millicores)"), 2000), { fieldtype: "Column Break" }, diff --git a/atlas/service/doctype/cargo_server/cargo_server.py b/atlas/service/doctype/cargo_server/cargo_server.py index 7300fbd37..287582d76 100644 --- a/atlas/service/doctype/cargo_server/cargo_server.py +++ b/atlas/service/doctype/cargo_server/cargo_server.py @@ -112,7 +112,9 @@ def reset_pilot_admin_password(self) -> dict[str, str]: password = generate_installer_password() virtual_machine = frappe.get_doc("Virtual Machine", cargo_server.virtual_machine) - result = SSHRunner(virtual_machine.ssh_host).run_command( + result = SSHRunner( + virtual_machine.ssh_host, proxy_command=virtual_machine.get_ssh_proxy_command() + ).run_command( PILOT_ADMIN_PASSWORD_RESET_COMMAND, data={"PILOT_ADMIN_PASSWORD": password}, ) @@ -208,10 +210,6 @@ def _validate_provision_request(self, values: dict[str, Any]) -> None: ): frappe.throw(_("Select a System Virtual Machine Image.")) - address = values.get("public_ipv4") - if not isinstance(address, str) or not address.strip(): - frappe.throw(_("Select a reserved public IPv4 allocation.")) - def _create_virtual_machine(self, values: dict[str, Any]) -> bool: from atlas.vm.core.vm_service import VirtualMachineCreateError, VirtualMachineService @@ -225,7 +223,6 @@ def _create_virtual_machine(self, values: dict[str, Any]) -> bool: "is_termination_protected": True, "hostname": "cargo", "ssh_keys": frappe.get_single("Atlas Settings").public_ssh_key, - "public_ipv4": values["public_ipv4"], } try: result = VirtualMachineService.create(request) @@ -246,7 +243,9 @@ def _set_pilot_release_tracker(self, enabled: bool) -> None: virtual_machine = frappe.get_doc("Virtual Machine", self.virtual_machine) command = "enable-pilot-release-tracker" if enabled else "disable-pilot-release-tracker" - result = SSHRunner(virtual_machine.ssh_host).run_command( + result = SSHRunner( + virtual_machine.ssh_host, proxy_command=virtual_machine.get_ssh_proxy_command() + ).run_command( PILOT_RELEASE_TRACKER_COMMAND, data={"CARGO_SITE": self.domain, "PILOT_RELEASE_TRACKER_COMMAND": command}, ) diff --git a/atlas/service/doctype/cargo_server/test_cargo_server.py b/atlas/service/doctype/cargo_server/test_cargo_server.py index dee47048a..6172cf4d7 100644 --- a/atlas/service/doctype/cargo_server/test_cargo_server.py +++ b/atlas/service/doctype/cargo_server/test_cargo_server.py @@ -16,7 +16,6 @@ "cpu_millicores": 2000, "memory_mib": 4096, "disk_mib": 16384, - "public_ipv4": "32eb57bc-9548-4a89-8358-543e26883569", } @@ -48,16 +47,7 @@ def test_provisioning_requires_a_system_image(self) -> None: ): CargoServer._validate_provision_request(server, VALID_REQUEST) - def test_provisioning_requires_a_public_address(self) -> None: - server = SimpleNamespace(virtual_machine=None, status="Not Provisioned") - with ( - patch.object(cargo_server_module.frappe.db, "exists", return_value=True), - patch.object(cargo_server_module.frappe.db, "get_value", return_value="system"), - self.assertRaisesRegex(frappe.ValidationError, "reserved public IPv4"), - ): - CargoServer._validate_provision_request(server, {**VALID_REQUEST, "public_ipv4": " "}) - - def test_virtual_machine_request_uses_the_reserved_public_address(self) -> None: + def test_virtual_machine_request_has_no_public_address(self) -> None: server = SimpleNamespace(virtual_machine=None, status="Pending", failure_message=None) virtual_machine_service = MagicMock() virtual_machine_service.create.return_value = {"name": "vm-00001", "is_draft": False} @@ -79,7 +69,7 @@ def test_virtual_machine_request_uses_the_reserved_public_address(self) -> None: self.assertTrue(request["is_termination_protected"]) self.assertNotIn("routes", request) self.assertEqual(request["hostname"], "cargo") - self.assertEqual(request["public_ipv4"], "32eb57bc-9548-4a89-8358-543e26883569") + self.assertNotIn("public_ipv4", request) self.assertEqual(request["cpu_millicores"], 2000) self.assertEqual(request["memory_mib"], 4096) self.assertEqual(request["disk_mib"], 16384) diff --git a/atlas/service/doctype/ipv6_router_server/ipv6_router_server.py b/atlas/service/doctype/ipv6_router_server/ipv6_router_server.py index 40da70fb6..3b9376131 100644 --- a/atlas/service/doctype/ipv6_router_server/ipv6_router_server.py +++ b/atlas/service/doctype/ipv6_router_server/ipv6_router_server.py @@ -167,7 +167,6 @@ def _create_virtual_machine(self, values: dict[str, Any]) -> bool: "is_termination_protected": True, "hostname": self.name, "ssh_keys": frappe.get_single("Atlas Settings").public_ssh_key, - "public_ipv4": values["public_ipv4"], } try: result = VirtualMachineService.create(request) @@ -206,7 +205,6 @@ def _validate_create_request(values: dict[str, Any]) -> None: "is_termination_protected": True, "hostname": "ipv6-router", "ssh_keys": frappe.get_single("Atlas Settings").public_ssh_key, - "public_ipv4": values.get("public_ipv4"), } try: request = VirtualMachineCreateRequest.from_value(virtual_machine_request) @@ -219,19 +217,9 @@ def _validate_create_request(values: dict[str, Any]) -> None: frappe.throw(_("Select a System Virtual Machine Image.")) image.validate_compatibility(request.disk_mib) - _validate_ipv4_allocation(request.public_ipv4) _validate_ipv6_pool(values.get("public_ip_pool")) -def _validate_ipv4_allocation(allocation_name: object) -> None: - """Require a free IPv4 allocation reserved by tenant 0.""" - if not isinstance(allocation_name, str) or not frappe.db.exists("Public IP Allocation", allocation_name): - frappe.throw(_("Select a reserved public IPv4 allocation.")) - allocation = frappe.get_doc("Public IP Allocation", allocation_name) - if allocation.version != "4" or allocation.status != "Reserved" or allocation.tenant_id != 0: - frappe.throw(_("Select a free public IPv4 allocation reserved by tenant 0.")) - - def _validate_ipv6_pool(pool_name: object) -> None: """Require a free IPv6 pool that can use the router address layout.""" if not isinstance(pool_name, str) or not frappe.db.exists("Public IP Pool", pool_name): diff --git a/atlas/service/doctype/ipv6_router_server/ipv6_router_server_list.js b/atlas/service/doctype/ipv6_router_server/ipv6_router_server_list.js index 8f7e0773e..c422943af 100644 --- a/atlas/service/doctype/ipv6_router_server/ipv6_router_server_list.js +++ b/atlas/service/doctype/ipv6_router_server/ipv6_router_server_list.js @@ -35,20 +35,6 @@ function showCreateIPv6RouterServerDialog() { reqd: 1, default: 8192, }, - { - fieldname: "public_ipv4", - fieldtype: "Link", - label: __("Public IPv4 Allocation"), - options: "Public IP Allocation", - reqd: 1, - description: __("Atlas uses this address for SSH."), - filters: { - status: "Reserved", - version: "4", - tenant_id: 0, - virtual_machine: ["is", "not set"], - }, - }, { fieldname: "public_ip_pool", fieldtype: "Link", diff --git a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py index 3c1a0d0e7..e77885c3c 100644 --- a/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py +++ b/atlas/service/doctype/ipv6_router_server/test_ipv6_router_server.py @@ -25,19 +25,6 @@ def public_ip_pool(**values) -> SimpleNamespace: ) -def ipv4_allocation(**values) -> SimpleNamespace: - return SimpleNamespace( - **( - { - "version": "4", - "status": "Reserved", - "tenant_id": 0, - } - | values - ) - ) - - class TestIPv6PoolValidation(UnitTestCase): def validate(self, pool: SimpleNamespace, has_allocations: bool = False) -> None: def exists(doctype, *_args, **_kwargs): @@ -65,26 +52,6 @@ def test_a_small_pool_is_refused(self) -> None: self.validate(public_ip_pool(prefix="2001:db8::/96")) -class TestIPv4AllocationValidation(UnitTestCase): - def validate(self, allocation: SimpleNamespace) -> None: - with ( - patch.object(router_module.frappe.db, "exists", return_value=True), - patch.object(router_module.frappe, "get_doc", return_value=allocation), - ): - router_module._validate_ipv4_allocation("allocation-1") - - def test_a_tenant_zero_reservation_is_accepted(self) -> None: - self.validate(ipv4_allocation()) - - def test_an_unreserved_allocation_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "reserved by tenant 0"): - self.validate(ipv4_allocation(status="Available")) - - def test_another_tenant_is_refused(self) -> None: - with self.assertRaisesRegex(frappe.ValidationError, "reserved by tenant 0"): - self.validate(ipv4_allocation(tenant_id=7)) - - class TestIPv6RouterCreation(UnitTestCase): def test_router_stores_the_mesh_address_of_its_virtual_machine(self) -> None: router_server = SimpleNamespace() diff --git a/atlas/service/doctype/proxy_server/proxy_server.py b/atlas/service/doctype/proxy_server/proxy_server.py index 55c6ae4e8..71f1e0a0e 100644 --- a/atlas/service/doctype/proxy_server/proxy_server.py +++ b/atlas/service/doctype/proxy_server/proxy_server.py @@ -9,10 +9,27 @@ from frappe import _ from frappe.model.document import Document +from atlas.atlas.core.mesh_address import get_region_mesh_address_prefix + if TYPE_CHECKING: from frappe.types import DF MAX_PROXY_SERVERS = 5 +ANYWHERE = ["0.0.0.0/0", "::/0"] + + +def get_proxy_firewall(region_id: int) -> dict[str, Any]: + """Admit HTTP and HTTPS from anywhere, and everything from regional services. Atlas reaches SSH through the host.""" + return { + "enabled": True, + "inbound": [ + {"protocol": "any", "cidrs": [f"{get_region_mesh_address_prefix(region_id)}::/64"]}, + {"protocol": "icmp", "cidrs": ANYWHERE}, + {"protocol": "tcp", "ports": "80", "cidrs": ANYWHERE}, + {"protocol": "tcp", "ports": "443", "cidrs": ANYWHERE}, + ], + "outbound": [{"protocol": "any", "cidrs": ANYWHERE}], + } class ProxyServer(Document): @@ -80,6 +97,7 @@ def create_virtual_machine(proxy_server: Any, values: dict[str, Any]) -> bool: """Create the virtual machine and update the Proxy Server state.""" from atlas.vm.core.vm_service import VirtualMachineCreateError, VirtualMachineService + settings = frappe.get_single("Atlas Settings") virtual_machine_request = { "virtual_machine_image": values.get("virtual_machine_image"), "cpu_millicores": values.get("cpu_millicores"), @@ -89,8 +107,9 @@ def create_virtual_machine(proxy_server: Any, values: dict[str, Any]) -> bool: "is_privileged": True, "is_termination_protected": True, "hostname": proxy_server.name, - "ssh_keys": frappe.get_single("Atlas Settings").public_ssh_key, + "ssh_keys": settings.public_ssh_key, "public_ipv4": values["public_ipv4"], + "firewall": get_proxy_firewall(settings.region_id), } try: result = VirtualMachineService.create(virtual_machine_request) diff --git a/atlas/service/doctype/proxy_server/test_proxy_server.py b/atlas/service/doctype/proxy_server/test_proxy_server.py index 67d2f7a33..0120ef26c 100644 --- a/atlas/service/doctype/proxy_server/test_proxy_server.py +++ b/atlas/service/doctype/proxy_server/test_proxy_server.py @@ -1,6 +1,7 @@ # Copyright (c) 2026, Frappe and Contributors # See license.txt +import ipaddress from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import atlas.service.core.proxy.configuration as configuration_module import atlas.service.doctype.proxy_server.proxy_server as proxy_server_module +from atlas.vm.core.models import FirewallConfiguration class IntegrationTestProxyServer(IntegrationTestCase): @@ -198,7 +200,7 @@ def test_the_creation_api_creates_a_proxy_for_the_new_vm(self) -> None: patch.object( proxy_server_module.frappe, "get_single", - return_value=SimpleNamespace(public_ssh_key="ssh-ed25519 AAAA atlas"), + return_value=SimpleNamespace(public_ssh_key="ssh-ed25519 AAAA atlas", region_id=1), ), patch("atlas.vm.core.vm_service.VirtualMachineService", virtual_machine_service), ): @@ -227,6 +229,24 @@ def test_the_creation_api_creates_a_proxy_for_the_new_vm(self) -> None: self.assertEqual(virtual_machine_service.create.call_args.args[0]["disk_mib"], 16384) proxy_server.enqueue_provisioning.assert_called_once() + def test_the_proxy_firewall_admits_ssh_only_from_regional_services(self) -> None: + firewall = FirewallConfiguration.from_value(proxy_server_module.get_proxy_firewall(1)) + + def admits(port: int, source: str) -> bool: + address = ipaddress.ip_address(source) + return any( + rule.protocol in {"any", "tcp"} + and (not rule.ports or rule.ports == str(port)) + and any(address in ipaddress.ip_network(cidr) for cidr in rule.cidrs) + for rule in firewall.inbound + ) + + self.assertTrue(admits(443, "203.0.113.1")) + self.assertTrue(admits(22, "fdaa:1::52")) + self.assertFalse(admits(22, "203.0.113.1")) + self.assertFalse(admits(22, "fdaa:1:0:7::1")) + self.assertTrue(admits(443, "fdaa:1:0:7::1")) + def test_the_record_is_committed_before_the_machine_request(self) -> None: proxy_server = MagicMock(name="proxy_server") proxy_server.name = "proxy-001" @@ -245,7 +265,7 @@ def test_the_record_is_committed_before_the_machine_request(self) -> None: patch.object( proxy_server_module.frappe, "get_single", - return_value=SimpleNamespace(public_ssh_key="ssh-ed25519 AAAA atlas"), + return_value=SimpleNamespace(public_ssh_key="ssh-ed25519 AAAA atlas", region_id=1), ), patch.object(proxy_server_module.frappe.db, "commit", side_effect=lambda: calls.append("commit")), patch("atlas.vm.core.vm_service.VirtualMachineService", virtual_machine_service), diff --git a/atlas/vm/SPEC.md b/atlas/vm/SPEC.md index 4b30157e0..0fe3c577c 100644 --- a/atlas/vm/SPEC.md +++ b/atlas/vm/SPEC.md @@ -32,6 +32,7 @@ This file identifies code owners and rules that a code change must preserve. The - A `within` rule reads the VMs of every host in the group of the candidate host. Placement locks each of those hosts without waiting, and keeps the locks until the draft commits. - Atlas changes one network value, then sends the complete network object to Metal. Public address requests own their corresponding default routes. - Guest-specific keys, metadata, and mesh addresses go through Metal and guest metadata. Do not bake them into a shared image. +- Atlas SSH to a guest runs `ip netns exec metal- nc 172.16.0.2 22` on the current host. It depends on the Metal namespace name and guest address in `linux_allocator.go`. - A protected VM cannot be terminated. An unprotected Atlas record is deleted only after Metal confirms that the VM is absent. - `ConsoleSession.close` owns console cleanup. Do not log Metal credentials. diff --git a/atlas/vm/core/metal_client.py b/atlas/vm/core/metal_client.py index 0a93506e3..5f180b858 100644 --- a/atlas/vm/core/metal_client.py +++ b/atlas/vm/core/metal_client.py @@ -65,29 +65,23 @@ def __init__(self, server: "MetalServer") -> None: @classmethod def get_api_url(cls, server: "MetalServer") -> str: - """Return the selected Metald IPv4 address for one server.""" - if server.settings.use_public_ip_for_metald: - label, address = "public IPv4 address", server.public_ipv4_address - else: - label, address = "private IPv4 address", server.private_ipv4_address - if not address: - raise MetalClientError(f"Server {server.name} has no {label}") - try: - ipv4_address = ipaddress.IPv4Address(address) - except (ipaddress.AddressValueError, TypeError) as error: - raise MetalClientError(f"Server {server.name} has an invalid {label}") from error - return f"https://{ipv4_address}:{cls.api_port}" + """Return the Metal control address on the server wg0.""" + return f"https://[{cls.get_wireguard_address(server)}]:{cls.api_port}" @classmethod def get_coordination_url(cls, server: "MetalServer") -> str: """Return the Metal coordination address for one server.""" + return f"https://[{cls.get_wireguard_address(server)}]:{cls.coordination_port}" + + @staticmethod + def get_wireguard_address(server: "MetalServer") -> ipaddress.IPv6Address: + """Return the validated wg0 address of one server.""" if not server.wireguard_ip_address: raise MetalClientError(f"Server {server.name} has no WireGuard IP address") try: - wireguard_address = ipaddress.IPv6Address(server.wireguard_ip_address) + return ipaddress.IPv6Address(server.wireguard_ip_address) except (ipaddress.AddressValueError, TypeError) as error: raise MetalClientError(f"Server {server.name} has an invalid WireGuard IP address") from error - return f"https://[{wireguard_address}]:{cls.coordination_port}" def get_console_connection(self, virtual_machine_id: str, mode: str = "tty") -> dict[str, str]: """Return the websocket URL for a VM console. The bridge holds the client certificate.""" diff --git a/atlas/vm/core/test_metal_client.py b/atlas/vm/core/test_metal_client.py index b391ca75b..c3b99a635 100644 --- a/atlas/vm/core/test_metal_client.py +++ b/atlas/vm/core/test_metal_client.py @@ -338,46 +338,22 @@ def virtual_machine_response() -> dict: class TestMetalClientConnection(UnitTestCase): @patch("atlas.vm.core.metal_client.client_certificate_files", return_value=("atlas.crt", "atlas.key")) @patch("atlas.vm.core.metal_client.ca_file", return_value="ca.crt") - def test_client_uses_the_validated_private_ipv4_address_by_default( - self, _ca_file: Mock, _certificate_files: Mock - ) -> None: - server = SimpleNamespace( - name="Server-1", - private_ipv4_address="10.0.0.2", - settings=SimpleNamespace(use_public_ip_for_metald=False), - ) + def test_client_uses_the_server_wireguard_address(self, _ca_file: Mock, _certificate_files: Mock) -> None: + server = SimpleNamespace(name="Server-1", wireguard_ip_address="fdab:1::12") client = MetalClient(server) - self.assertEqual(client.base_url, "https://10.0.0.2:9000") + self.assertEqual(client.base_url, "https://[fdab:1::12]:9000") self.assertEqual(client.ca_file, "ca.crt") self.assertEqual(client.client_certificate, ("atlas.crt", "atlas.key")) - @patch("atlas.vm.core.metal_client.client_certificate_files", return_value=("atlas.crt", "atlas.key")) - @patch("atlas.vm.core.metal_client.ca_file", return_value="ca.crt") - def test_client_uses_the_public_ipv4_address_when_configured( - self, _ca_file: Mock, _certificate_files: Mock - ) -> None: - server = SimpleNamespace( - name="Server-1", - public_ipv4_address="203.0.113.8", - settings=SimpleNamespace(use_public_ip_for_metald=True), - ) - client = MetalClient(server) - - self.assertEqual(client.base_url, "https://203.0.113.8:9000") - @patch("atlas.vm.core.metal_client.client_certificate_files") @patch("atlas.vm.core.metal_client.ca_file") - def test_client_rejects_an_invalid_private_ipv4_address( + def test_client_rejects_an_invalid_wireguard_address( self, ca_file_mock: Mock, certificate_files: Mock ) -> None: - server = SimpleNamespace( - name="Server-1", - private_ipv4_address="not-an-address", - settings=SimpleNamespace(use_public_ip_for_metald=False), - ) + server = SimpleNamespace(name="Server-1", wireguard_ip_address="10.0.0.2") - with self.assertRaisesRegex(MetalClientError, "invalid private IPv4 address"): + with self.assertRaisesRegex(MetalClientError, "invalid WireGuard IP address"): MetalClient(server) ca_file_mock.assert_not_called() @@ -607,7 +583,7 @@ def test_repeatable_calls_report_an_uncertain_transport_failure(self) -> None: self.assertTrue(caught.exception.uncertain) def test_api_url_rejects_a_server_without_an_address(self) -> None: - server = Mock(private_ipv4_address="", settings=SimpleNamespace(use_public_ip_for_metald=False)) + server = Mock(wireguard_ip_address="") server.name = "metal-1" with self.assertRaises(MetalClientError): diff --git a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py index f1083a60f..d9b64713b 100644 --- a/atlas/vm/doctype/virtual_machine/test_virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/test_virtual_machine.py @@ -378,6 +378,21 @@ def test_new_document_reads_virtual_fields_without_a_server(self) -> None: self.assertEqual(virtual_machine.current_state, "unknown") self.assertIsNone(virtual_machine.desired_state) + def test_guest_ssh_goes_through_the_current_host_namespace(self) -> None: + """A migrated VM has a new host, so Atlas reads it for each connection.""" + virtual_machine = frappe.new_doc("Virtual Machine") + virtual_machine.name = "vm-0000042" + virtual_machine.server = "metal-2" + + with patch.object( + virtual_machine_module.frappe, "get_doc", return_value=SimpleNamespace(ssh_host="fdab:1::2") + ) as get_doc: + proxy_command = virtual_machine.get_ssh_proxy_command() + + get_doc.assert_called_once_with("Metal Server", "metal-2") + self.assertIn("root@fdab:1::2", proxy_command) + self.assertIn("ip netns exec metal-vm-0000042 nc 172.16.0.2 22", proxy_command) + class TestVirtualMachineResize(UnitTestCase): def build_virtual_machine(self, *, is_terminating: int = 0) -> VirtualMachine: diff --git a/atlas/vm/doctype/virtual_machine/virtual_machine.py b/atlas/vm/doctype/virtual_machine/virtual_machine.py index a3033dccc..e175b7c17 100644 --- a/atlas/vm/doctype/virtual_machine/virtual_machine.py +++ b/atlas/vm/doctype/virtual_machine/virtual_machine.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import shlex from typing import Any import frappe @@ -12,6 +13,7 @@ from atlas.atlas.core.background_jobs import run_as_admin from atlas.atlas.core.exceptions import AtlasConflictError, AtlasUserError from atlas.atlas.core.parsing import strict_bool +from atlas.atlas.core.ssh import SSHRunner from atlas.atlas.core.tags import validate_tags from atlas.atlas.doctype.ssh_task.ssh_task import delete_tasks_for_target from atlas.vm.core import reconciliation @@ -28,6 +30,8 @@ # Atlas WG Mesh reserves tenant 0 for the privileged tenant. PRIVILEGED_TENANT_ID = 0 IMAGE_TYPES = ("machine", "system") +# The guest address inside the Metal namespace metal-. +GUEST_IP_ADDRESS = "172.16.0.2" class VirtualMachine(Document): @@ -197,13 +201,18 @@ def routes(self) -> str: @property def ssh_host(self) -> str: - """Return the address an SSH Task connects to.""" - if not self.public_ipv4: - frappe.throw( - _("Virtual Machine {0} has no public IPv4 address. Attach one first.").format(self.name) - ) + """Return the SSH host name. The proxy command carries the connection.""" + return self.name - return self.public_ipv4 + def get_ssh_proxy_command(self) -> str: + """Reach guest SSH from the VM network namespace on its current host, through the host wg0.""" + if not self.server: + frappe.throw(_("Virtual Machine {0} has no Metal Server.").format(self.name)) + + server = frappe.get_doc("Metal Server", self.server) + return SSHRunner(server.ssh_host).get_proxy_command( + f"ip netns exec {shlex.quote('metal-' + self.name)} nc {GUEST_IP_ADDRESS} 22" + ) @property def disk_throughput_mibps(self) -> int: diff --git a/docs/develop/atlas-app.md b/docs/develop/atlas-app.md index a90b7c02c..52529dc38 100644 --- a/docs/develop/atlas-app.md +++ b/docs/develop/atlas-app.md @@ -103,10 +103,10 @@ Pass validated values to the domain service. The first docstring line becomes th Atlas builds `metald` and the WG Mesh CLI after installation and migration. A build runs only when its source changes. Atlas publishes each build as a public File, stores the File link in Atlas Settings, and keeps earlier files available. -A host downloads the binary during `install-metald.sh`, so the file needs an address that the host can reach. Set `atlas_base_url` in the site configuration for that address. Atlas uses the site URL when the key is absent. +A host downloads the binary during `install-metald.sh`, so the file needs an address that the host can reach. Set `atlas_internal_url` in the site configuration to the Atlas listener on its mesh address. Atlas uses `atlas_base_url`, then the site URL, when the key is absent. See [Atlas access to hosts](../region/host-access.md#internal-url). ```json -"atlas_base_url": "https://devfc2.example.com" +"atlas_internal_url": "http://[fdaa:1::ffff:ffff:ffff:ffff]:8000" ``` Install the build tools before you install or migrate Atlas. `make` runs both builds. `clang`, `libbpf-dev`, and `linux-libc-dev` build the WG Mesh eBPF object. diff --git a/docs/develop/test-region.md b/docs/develop/test-region.md index 6da98af52..4d03df1d0 100644 --- a/docs/develop/test-region.md +++ b/docs/develop/test-region.md @@ -8,7 +8,7 @@ This guide builds a small test region: one Atlas site, one host, and one VM. Use Keep the Frappe worker active. Provider setup, catalog sync, and Metal Server provisioning run in background jobs. -Use [`atlas-vm`](../../scripts/atlas-vm/) for an automatic installation in a Firecracker VM. It installs Atlas and completes the settings, provider, DNS, catalog, and system image steps. It does not create a Metal Server. +Use [`atlas-vm`](../../scripts/atlas-vm/) for an automatic installation in a Firecracker VM. It installs Atlas and completes the settings, provider, DNS, catalog, and system image steps. It imports its host as a Metal Server only when `atlas.import_server_id` is set. ## Before you start @@ -38,7 +38,7 @@ pilot --site atlas.localhost install-app atlas ## 3. Set a public Atlas address -For now, a Metal Server must reach the Atlas site to download `metald` and WG Mesh during installation. Use a public Cloudflare Tunnel or ngrok URL for local development. +Central and external clients need a public Atlas address. Use a public Cloudflare Tunnel or ngrok URL for local development. Cargo and hosts use the internal URL after step 5. Set the URL as `atlas_base_url`: @@ -72,6 +72,15 @@ In **Atlas Settings**, select **Actions** and click these buttons: Wait until both actions finish. Atlas marks the settings as complete after both actions succeed. +Connect this machine to the host network. Run: + +```sh +pilot --site configure-atlas-wireguard +sudo ATLAS_WIREGUARD_TCP_PORTS="22, 80, 443, 2222, 8000" scripts/install-atlas-wireguard.sh /sites//private/wireguard/atlas0.conf +``` + +Start a listener on the Atlas mesh address that sends `Host: ` to `127.0.0.1:8000`, for example nginx. Set it as `atlas_internal_url`, for example `http://[fdaa:1::ffff:ffff:ffff:ffff]:8000`. Your machine is outside the provider network, so it needs the [development gateway](../region/host-access.md#development-gateway) before it can reach hosts through WireGuard. + ## 6. Sync the Metal Server catalog Open **Metal Server Size** and click **Sync**. Then open **Metal Server Image** and click **Sync**. @@ -82,7 +91,11 @@ Wait for the background jobs to finish. The catalog supplies the provider size a Open **Metal Server** and create a record. Select `EM-A116X-SSD` and an Ubuntu 24.04 Metal Server Image. -Save the record. Wait for its status to become `Running`. Open its linked **SSH Task** records to see each host command and its result. +Save the record. The first host can stop with status `Failed` at `wireguard-link` after 120 seconds. + +If it does, [deploy the development gateway](../region/host-access.md#development-gateway) with `--ssh-host` and the host's public IPv4 address. Then use **Setup Metal Server** to retry. + +Wait until the host is `Running`. Open its linked **SSH Task** records to see each host command and its result. ## 8. Build a VM image diff --git a/docs/interfaces/security.md b/docs/interfaces/security.md index f9e1e6bca..5d06afce3 100644 --- a/docs/interfaces/security.md +++ b/docs/interfaces/security.md @@ -29,7 +29,7 @@ Metal uses TLS 1.3 with no shared secret. | Coordination | `9001` | Any regional node certificate. Source-side migration routes only. | | One-shot snapshot stream | `9002` | Any regional node certificate. | -Coordination and snapshot listeners bind only to the WireGuard address. The destination also checks the source server certificate against the source address. +All three listeners bind only to the WireGuard address. The host firewall admits `9000` only from the Atlas WireGuard address and `9001` and `9002` only from host addresses. See [Atlas access to hosts](../region/host-access.md#host-firewall). The destination also checks the source server certificate against the source address. ### Certificate lifetime diff --git a/docs/networking/http-proxy/high-availability.md b/docs/networking/http-proxy/high-availability.md index a87d0a4be..56ebccaab 100644 --- a/docs/networking/http-proxy/high-availability.md +++ b/docs/networking/http-proxy/high-availability.md @@ -4,6 +4,8 @@ The HTTP proxy cluster has 1 to 5 nodes in one region. Each node serves traffic Atlas manages the nodes, Domain Name System (DNS) records, peer membership, certificates, and cluster passwords. See the [control daemon guide](control-daemon.md) for the public API. +Peers talk over the VM mesh. Atlas writes each peer name with its mesh address into `/etc/hosts` on every node. + ## Addresses and health Atlas publishes node, regional, and wildcard names in that order. [Proxy provisioning](provisioning.md#which-dns-names-atlas-publishes) lists their records and TTLs. Peers use stable node addresses. Public clients use regional DNS. diff --git a/docs/networking/index.md b/docs/networking/index.md index 96d3d5840..2f8ddafc8 100644 --- a/docs/networking/index.md +++ b/docs/networking/index.md @@ -34,6 +34,8 @@ Each host has a WireGuard interface, `wg0`, with an address in `fdab::/16`. Ever The Atlas app keeps the list of hosts and their keys. It sends the complete peer list to each host during [host sync](../region/host-sync.md), and Metal applies it. A new host joins the mesh at the next sync. +Atlas itself is one more `wg0` peer on each host, outside the mesh peer list. Hosts can reach Atlas. Among VMs, only tenant-0 VMs can reach its mesh address. [Atlas access to hosts](../region/host-access.md) explains the path. + ## Layer 3: WG Mesh Each VM gets a stable address. The Atlas app derives it when it creates the VM: diff --git a/docs/networking/ipv6-router.md b/docs/networking/ipv6-router.md index 06d789530..cc84b41c6 100644 --- a/docs/networking/ipv6-router.md +++ b/docs/networking/ipv6-router.md @@ -77,7 +77,7 @@ The router checks address shape and packet format. WG Mesh checks whether the ta ## Prepare the router VM -Atlas creates an [IPv6 Router Server service VM](../region/service-vms.md). It waits until the VM leaves draft state and Metal has applied its public IPv4 address. This wait keeps the router's full network update from overlapping the IPv4 update. +Atlas creates an [IPv6 Router Server service VM](../region/service-vms.md) with no public IPv4 address. It waits until the VM leaves draft state and Metal holds the VM. The setup job then makes the VM a network gateway, attaches the public IPv6 pool through the provider, and gives the router a `2000::/3` route through its host. It waits for SSH, installs the hashed router package, and sets the service record to `Active`. diff --git a/docs/networking/wg-mesh/index.md b/docs/networking/wg-mesh/index.md index c4985090b..8135c6567 100644 --- a/docs/networking/wg-mesh/index.md +++ b/docs/networking/wg-mesh/index.md @@ -32,6 +32,7 @@ flowchart LR | `local_vms` | VM address to local interface. | | `remote_vms` | Remote VM address to host WireGuard address. Least recently used entries are evicted. | | `privileged_vms` | Tenant-0 addresses that can reach every tenant. | +| `controller_address` | The Atlas tenant-0 address. The VM hook passes traffic to it to Linux, which routes it through `wg0`. | | `peer_list` | Peer IPv4, MAC, and WireGuard addresses. | | `discovery_limits` | NDP request limit for each VM interface. | | `gateways` | Interfaces of the gateway VMs on this host. | @@ -51,10 +52,11 @@ The VM hook checks each packet a VM sends, in this order: 2. Send a destination that has a gateway route to its gateway. 3. Drop a packet whose source address the VM does not own. 4. Drop a packet to another tenant, unless one side is privileged. -5. Drop a foreign source to a local VM that has no gateway route back to that source. -6. Leave local delivery to Linux. -7. Tunnel a known remote destination through WireGuard. -8. Start an NDP lookup for an unknown destination. Each interface can start 10 lookups a second, with a burst of 50. +5. Leave a VM-sourced packet to the Atlas controller address to Linux, which routes it through `wg0`. +6. Drop a foreign source to a local VM that has no gateway route back to that source. +7. Leave local delivery to Linux. +8. Tunnel a known remote destination through WireGuard. +9. Start an NDP lookup for an unknown destination. Each interface can start 10 lookups a second, with a burst of 50. ## Lookup (NDP) diff --git a/docs/networking/wg-mesh/operations.md b/docs/networking/wg-mesh/operations.md index 2ae3f5e40..49893a4f1 100644 --- a/docs/networking/wg-mesh/operations.md +++ b/docs/networking/wg-mesh/operations.md @@ -21,9 +21,11 @@ The built command does not need Clang or bpftool on the target host. ## Configure a host ```sh -sudo atlas-wg-mesh configure --uplink eno1.1680 --wireguard wg0 +sudo atlas-wg-mesh configure --uplink eno1.1680 --wireguard wg0 --controller fdaa:1::ffff:ffff:ffff:ffff ``` +`--controller` sets the Atlas mesh address. Without it, the stored address stays. + The uplink needs IPv4, IPv6, and an Ethernet MAC address. The WireGuard interface needs an address in `fdab::/16`. diff --git a/docs/operate/atlas.md b/docs/operate/atlas.md index 7fba33641..b54861c2c 100644 --- a/docs/operate/atlas.md +++ b/docs/operate/atlas.md @@ -49,7 +49,7 @@ A VM read fails, or Metal Server synchronization writes a Metal connection Error **Check** 1. Open the Metal Server document. -2. Check its Metal address and the `use_public_ip_for_metald` setting. +2. Run `wg show atlas0` on the Atlas machine and check the handshake with the host. See [Atlas access to hosts](../region/host-access.md#recovery). 3. Check recent Error Logs. 4. On the host, run `systemctl status metal.service` and `journalctl -u metal.service --since "15 minutes ago"`. diff --git a/docs/region/cargo.md b/docs/region/cargo.md index f03be1462..be3049e3c 100644 --- a/docs/region/cargo.md +++ b/docs/region/cargo.md @@ -11,7 +11,7 @@ flowchart LR ## Before you provision -Use the **Provision** action on Cargo Server. Select an Available System image and a reserved tenant-0 IPv4 allocation. An Active HTTP proxy must already exist because Cargo's public routes go through it. +Use the **Provision** action on Cargo Server. Select an Available System image. Cargo needs no public IPv4 address. An Active HTTP proxy must already exist because Cargo's public routes go through it. | Resource | Provision form default | | --- | --- | @@ -25,7 +25,7 @@ The storage-node count must be at least the replication factor. The [service VM Atlas waits for the VM to leave draft state. A job then runs these steps: -1. Wait for root SSH on the VM's public IPv4 address. +1. Wait for root SSH through the VM host. 2. Run `install-cargo.sh` in a synchronous SSH Task. 3. Point `cargo` and `cargo-pilot` proxy routes at the VM's mesh IPv6 address. 4. Call Cargo's ping endpoint through the regional proxy and expect `pong`. diff --git a/docs/region/configuration.md b/docs/region/configuration.md index d12818ff3..f2deb04c1 100644 --- a/docs/region/configuration.md +++ b/docs/region/configuration.md @@ -1,13 +1,15 @@ # Atlas configuration -Use **Atlas Settings** for regional configuration. Set the site value `atlas_base_url` to an address hosts can reach for Atlas files. +Use **Atlas Settings** for regional configuration. Set the site value `atlas_internal_url` to the Atlas listener on its mesh address. Hosts and tenant-0 VMs reach Atlas through it. + +Set `atlas_base_url` to the public Atlas address when the site URL is not public. ## Settings by purpose | Group | What it controls | Important effect | | --- | --- | --- | | Region and provider | Region identity, provider adapter, host catalog defaults, and credentials. | New hosts and signed regional identities use these values. | -| Host and network | Private network range, MTU, unicast mesh mode, and Metal endpoint choice. | Host setup and sync use these values. | +| Host and network | Private network range, MTU, unicast mesh mode, and Atlas WireGuard identity. | Host setup and sync use these values. | | Placement | Strategy, sleepy VM pool, overcommit factor, and host auto-spawn. | New VM capacity checks and host expansion use these values. | | Trust | Metal certificate authority, Atlas client certificate, regional signing key, and Central public key source. | Atlas, Metal, and service clients use these credentials. | | Proxy and DNS | Wildcard domain, certificate, proxy password, and DNS access. | Proxy nodes receive updated configuration. | @@ -31,14 +33,18 @@ Atlas creates regional signing and Metal trust material as part of settings setu | Network setting | Purpose | | --- | --- | -| `use_public_ip_for_metald` | Selects the Atlas-facing Metal endpoint where supported. It leaves node-to-node WireGuard unchanged. | +| `atlas_internal_url` | Site URL that hosts and tenant-0 VMs use for Atlas files and APIs. | +| `atlas_base_url` | Public site URL when the normal site URL is not public. It is the fallback for internal clients. | +| `atlas_wireguard_mtu` | MTU of `atlas0`. Restart the interface after a change. | | `is_unicast_network_enabled` | Uses unicast discovery when the host network cannot carry multicast. | +The first three values are site configuration keys. See [Atlas access to hosts](host-access.md) for setup and recovery. + ## Limits and recovery **Saving a field does not prove that every host applied it.** Setup, sync, or service configuration may still need to run. -Verify hosts and services after changing region identity, trust material, storage credentials, or public-address mode. +Verify hosts and services after changing region identity, trust material, storage credentials, or the internal URL. ::: details Source code and tests diff --git a/docs/region/host-access.md b/docs/region/host-access.md new file mode 100644 index 000000000..679a37f73 --- /dev/null +++ b/docs/region/host-access.md @@ -0,0 +1,130 @@ +# How Atlas reaches hosts + +Atlas reaches each Metal host through WireGuard. SSH, the Metal control API, and the browser console use the host `wg0` interface. + +Hosts use this path for file downloads when `atlas_internal_url` is set. After installation, the host firewall rejects new management connections on the public interface. + +```text +Atlas atlas0 (fdaa:::ffff:ffff:ffff:ffff) ==WireGuard==> host wg0 (fdab::) + ├── sshd :22, metald :9000 + ├── tenant-0 VMs (fdaa:::) + └── ip netns exec metal- nc 172.16.0.2 22 (guest SSH) +hosts and tenant-0 VMs --http--> atlas_internal_url (an Atlas listener on its mesh address) +``` + +## The Atlas peer + +Atlas has one WireGuard identity for each region. **Atlas Settings** holds its address and public key. + +The address is a tenant-0 VM address, `fdaa:::ffff:ffff:ffff:ffff`. No VM can take it. WG Mesh trusts only `fdab` senders as hosts, so Atlas cannot send mesh tunnels. + +Hosts can reach Atlas. Among VMs, only tenant-0 VMs can reach its mesh address. + +Atlas Settings also holds the private key in an encrypted, hidden field, so a site backup keeps it. Atlas never copies it to a host. + +Atlas writes `sites//private/wireguard/atlas0.conf`. Each host peer allows the host `fdab` address and the tenant-0 VMs on that host. A scheduler job writes the file again within 10 seconds when a host or a tenant-0 VM changes, for example after a migration. + +A root systemd timer applies the file every 10 seconds. When the file changed, it copies it to `/etc/wireguard/atlas0.conf` and runs `wg syncconf`, or `wg-quick up` for the first start. A timer is used because SELinux can hide a home directory from a path unit. + +Atlas uses each host private IPv4 address as the endpoint. The Atlas VM reaches it through the masquerade on its parent host. A development Atlas reaches it through the [development gateway](#development-gateway). + +Site config `atlas_wireguard_mtu` sets the `atlas0` MTU. Use `1280` through the development gateway. + +## Set up the Atlas machine + +Run these commands once on the machine that runs the Atlas bench: + +```sh +pilot --site SITE configure-atlas-wireguard +sudo scripts/install-atlas-wireguard.sh /sites/SITE/private/wireguard/atlas0.conf +wg show atlas0 +``` + +`configure-atlas-wireguard` creates the identity once and writes the peer file. Run it before the first host setup, which refuses to start without it. The install script adds `atlas-wireguard-atlas0.timer` and `atlas-wireguard-atlas0.service`. + +The service also applies table `inet atlas_atlas0`. It admits TCP `22`, `80`, `443`, and `2222` on `atlas0`, plus ICMPv6 and replies. + +To change the TCP ports, run the install script again with `ATLAS_WIREGUARD_TCP_PORTS`. For a development listener on port `8000`, set the value to `"22, 80, 443, 2222, 8000"`. + +WireGuard needs no port on `atlas0`, because Atlas starts every handshake from the uplink. + +## Development gateway + +A development Atlas runs outside the provider network. The gateway is atlas-vm in gateway mode on one Metal host. It carries the `atlas0` packets to the host private endpoints, so hosts expose no WireGuard port publicly. + +```text +laptop atlas-gateway (172.16.100.3) ==outer WireGuard==> atlas-vm :51821 ── masquerade ──> 10.1.0.x:51820 +laptop atlas0 ======================= inner WireGuard, end to end ======================> host wg0 +``` + +1. Create the first Metal Server record. Its setup can stop at `wireguard-link` after 120 seconds. The host firewall is not installed yet. +2. Run `pilot --site SITE deploy-dev-gateway --ssh-host `. The command installs atlas-vm on that host and writes `atlas-gateway.conf`. When Atlas can reach that host later, omit `--ssh-host` to update the gateway. +3. Run `sudo scripts/install-atlas-wireguard.sh /sites/SITE/private/wireguard/atlas-gateway.conf` on the Atlas machine. +4. Set the Atlas MTU to `1280` with `pilot --site SITE set-config -p atlas_wireguard_mtu 1280`. Then run `pilot --site SITE configure-atlas-wireguard` to rewrite `atlas0.conf`. +5. Restart `atlas0` with `sudo wg-quick down atlas0 && sudo systemctl start atlas-wireguard-atlas0.service`. `wg syncconf` does not change the MTU of a running interface. +6. If the Metal Server status is `Failed`, use **Setup Metal Server** to retry. Normal setup installs Metal and the host firewall after the WireGuard link works. + +The gateway cannot decrypt the `atlas0` sessions. The outer link carries only the provider private network. + +## Host side + +`configure-wireguard.sh` adds the Atlas public key and address to the host `wg0.conf`. The Atlas peer has no endpoint. Atlas starts the handshake and keeps it alive. + +Metal applies only the host peers from [host sync](host-sync.md). It removes only the peers that it added, so the Atlas peer stays. `metald.toml` names the Atlas address as `[wg_mesh] controller_address`, and WG Mesh lets tenant-0 VMs send to it through `wg0`. + +`metald` binds its control API to the host `wg0` address. The host certificate names that address. + +## SSH + +| Target | Path | +| --- | --- | +| Host before its WireGuard setup | Public IPv4 address. Used only during provisioning. | +| Host after its WireGuard setup | Host `fdab` address. | +| Guest | SSH to the host, then `ip netns exec metal- nc 172.16.0.2 22` as the SSH `ProxyCommand`. | + +Provisioning has a `wireguard-link` step. It writes `atlas0.conf` with the new host at once, then waits for root SSH on the `fdab` address before it installs Metal. + +Atlas reads the VM host for each connection. The guest needs no public address for Atlas SSH. + +## Host firewall + +`install_metal` installs table `inet atlas_host` and the `atlas-host-firewall` unit. The rules cover traffic to the host itself. + +| Input | Allowed from | +| --- | --- | +| WireGuard UDP | The private network, and the local atlas-vm (`tap-atlas`). | +| SSH and Metal control `9000` | The Atlas address on `wg0`. | +| Migration `9001` and `9002` | Host addresses on `wg0`. | +| SSH for recovery | The private network on the private uplink. | +| ICMP, DHCP, and replies | Any address. | + +The Metal network setup, which `metal.service` runs at start, drops guest packets to the private network CIDR, on any interface. Guest public addresses, NAT, and mesh traffic do not change. + +## Internal URL + +Hosts and tenant-0 VMs download Atlas files from `atlas_internal_url`. Cargo uses it for the Atlas API and JWKS. Proxies use it for JWKS. + +Set it to an Atlas listener on the Atlas mesh address, for example `http://[fdaa:1::ffff:ffff:ffff:ffff]:8000`. The listener must send the Atlas site name in `Host`. If the port is not `80` or `443`, run the WireGuard install script with that port in `ATLAS_WIREGUARD_TCP_PORTS`. + +Without `atlas_internal_url`, clients use `atlas_base_url` or the site URL. + +## Recovery + +| Problem | Action | +| --- | --- | +| No handshake on `atlas0` | Check `wg show atlas0`, the gateway link, and UDP 51820 on the host. | +| New host stops at `wireguard-link` | Check that `atlas0.conf` lists the host and that the timer applied it: `systemctl status atlas-wireguard-atlas0.service`. | +| Atlas key lost | Restore the site backup with its `site_config.json`, because the encryption key is in that file. A new key needs the provider console on every host: clear `wireguard_public_key` in Atlas Settings, run `configure-atlas-wireguard`, then run `configure-wireguard.sh` on each host. | +| Host firewall blocks access | From the private network or console, run `systemctl disable --now atlas-host-firewall && nft delete table inet atlas_host`. | + +::: details Source code and tests + +- [Atlas peer](../../atlas/metal_server/core/atlas_peer.py) owns the identity and `atlas0.conf`. [Development gateway](../../atlas/metal_server/core/development_gateway.py) owns `atlas-gateway.conf`. +- [Timer installer](../../scripts/install-atlas-wireguard.sh) applies a file and its input filter as root. +- [WireGuard script](../../atlas/scripts/configure-wireguard.sh) adds the Atlas peer to a host. +- [Host firewall script](../../atlas/scripts/install-host-firewall.sh) writes the host rules. +- [atlas-vm](../../scripts/atlas-vm/atlas_vm.py) runs the gateway VM. +- [WG Mesh VM hook](../../services/wg-mesh/bpf/vm.h) routes tenant-0 traffic to the controller. +- [Atlas peer tests](../../atlas/metal_server/core/test_atlas_peer.py) and [provisioning tests](../../atlas/metal_server/core/test_provisioning.py) check identity and setup order. + +::: diff --git a/docs/region/hosts-and-providers.md b/docs/region/hosts-and-providers.md index da797b6a4..df7cca286 100644 --- a/docs/region/hosts-and-providers.md +++ b/docs/region/hosts-and-providers.md @@ -17,6 +17,16 @@ Atlas commits a `Pending` record before contacting the provider. The stable reco Setup prepares provider resources, SSH, networking, WireGuard, and Metal. It saves completed phases in MariaDB and marks the host `Running` only after completion. See the [provisioning sequence](index.md). +### Import a provider server + +Use **Import Server** on the Metal Server list for a Scaleway or AWS server that Atlas did not create. Enter its provider server ID. Atlas matches its size and image to the catalog, adds the Scaleway private network option when it is missing, and runs the normal setup. + +The Atlas SSH key must allow login as a user that the provider supports. A non-root user needs passwordless `sudo`. Atlas copies that user's authorized keys to root and removes the user during promotion. Use root login if the existing user must remain. + +Atlas uses the Scaleway `/dev/md2` array or the AWS storage volume mapped as `/dev/sdb`. If an imported AWS server has no volume mapped as `/dev/sdb`, enter a **Storage Pool Device** that exists on the host. You can also name a disk image file that you created. A new pool needs an empty device. Setup reuses an existing `metal` pool. + +Importing the same server again continues setup if it is incomplete. The command line equivalent is `pilot --site SITE import-metal-server [--storage-pool-device PATH]`. Generic hosts use **Add Server** instead. + ### Retry setup On failure, read the phase in the Error Log. Correct the cause, then use **Setup Metal Server**. Keep the provider identity so the retry can reuse the host. diff --git a/docs/region/index.md b/docs/region/index.md index a2daae9c2..749942d8a 100644 --- a/docs/region/index.md +++ b/docs/region/index.md @@ -8,9 +8,10 @@ A new Metal Server queues a background job: 1. Create or reuse the provider host. 2. Ask the provider to prepare it and wait for root SSH access. -3. Configure the provider network and WireGuard. -4. Install Metal, WG Mesh, TLS credentials, storage, and systemd units. -5. Mark the host `Running` and queue disk inventory sync. +3. Configure the provider network and WireGuard, with the Atlas peer on `wg0`. +4. Wait for root SSH on the host `wg0` address. +5. Install storage, TLS credentials, Metal, WG Mesh, systemd units, and the host firewall. +6. Mark the host `Running` and queue disk inventory sync. The provider adapter supplies host-specific operations. An adapter for a manually prepared host can expect some resources to exist already. @@ -29,7 +30,7 @@ The installation needs: - Atlas-built binaries. - A regional certificate. -Atlas writes the control address and a coordination listener on the WireGuard address. The install script creates the ZFS pool. `metald` does not select its device. +Atlas writes the control and coordination listeners on the WireGuard address. [Atlas access to hosts](host-access.md) explains the path. The install script creates the ZFS pool. `metald` does not select its device. ## Failure and recovery diff --git a/docs/region/metald.md b/docs/region/metald.md index 3e8de817f..511f30b4f 100644 --- a/docs/region/metald.md +++ b/docs/region/metald.md @@ -42,7 +42,7 @@ At startup, Metal compares running VM units with adopted console handles. If sys | Setting | Default | What to check | | --- | --- | --- | -| Control address | `127.0.0.1:8080` | Installed hosts need an Atlas-reachable address. A Unix socket is also accepted. | +| Control address | `127.0.0.1:8080` | Installation binds the host WireGuard address. See [Atlas access to hosts](host-access.md). A Unix socket is also accepted. | | Coordination address | `127.0.0.1:9001` | Use the host's WireGuard address when installed. | | Snapshot port | `9002` | Binds on the coordination host address, not a wildcard. | | `base_dir` | `/var/lib/metal` | Holds VM records and host state. | diff --git a/docs/region/service-vms.md b/docs/region/service-vms.md index 24d8d368c..9c36bcb14 100644 --- a/docs/region/service-vms.md +++ b/docs/region/service-vms.md @@ -12,7 +12,11 @@ Atlas owns each VM and its setup record. The software inside the VM owns its tra ## What Atlas creates -Reserve a **tenant-0 public IPv4 allocation** on the Public IP Pool form before you provision a service. Each service VM uses one. Atlas creates a privileged, termination-protected VM through the normal VM service and attaches that allocation. +Atlas creates a privileged, termination-protected VM through the normal VM service. Only the HTTP proxy needs a **tenant-0 public IPv4 allocation**. Reserve it on the Public IP Pool form first. + +The proxy guest firewall admits TCP 80 and 443 and ICMP from any address. It admits all traffic from tenant-0 mesh addresses in the region. + +Atlas reaches service SSH through the VM host, not a public address. See [Atlas access to hosts](host-access.md#ssh). The service record and VM have different states. For example, the VM can exist while its service record is still `Pending`. Do not infer service readiness from the VM state. @@ -28,7 +32,7 @@ Cargo also starts as `Not Provisioned`. See [Cargo recovery](cargo.md#operate-an ## Why a service can wait -Atlas queues a setup job after VM creation. If the VM is still a draft, the job leaves the service `Pending`. The scheduler queues it again. The router waits for Metal to apply its public IPv4 address as well, because its next step replaces the full network configuration. +Atlas queues a setup job after VM creation. If the VM is still a draft, the job leaves the service `Pending`. The scheduler queues it again. ::: info Check the service record first A `Pending` record does not need a second provision request. A `Failed` record does not automatically retry. Read its Failure field and the linked SSH Task before taking the service-specific recovery action. diff --git a/docs/storage/image-records.md b/docs/storage/image-records.md index 7bc7c7aeb..e6cb3be5f 100644 --- a/docs/storage/image-records.md +++ b/docs/storage/image-records.md @@ -36,7 +36,7 @@ The `artifact_storage` field selects the artifact location: ### Bootstrap without object storage -Build the first System image with `--storage site-file`. Set `atlas_base_url` to an address the host can reach. +Build the first System image with `--storage site-file`. Set `atlas_internal_url` to an address the host can reach. Site Files are public, so only System images can use them. The tenant download route refuses Site File images. diff --git a/metal/cmd/metald/SPEC.md b/metal/cmd/metald/SPEC.md index 84bdbeaf8..d41e84b2c 100644 --- a/metal/cmd/metald/SPEC.md +++ b/metal/cmd/metald/SPEC.md @@ -62,6 +62,7 @@ The default path is `/var/lib/metal/metald.toml`. A missing default file is perm | `wg_mesh.enabled` | `true` | Enables WG Mesh setup. | | `wg_mesh.binary_path` | `/usr/local/bin/atlas-wg-mesh` | WG Mesh CLI. | | `wg_mesh.uplink` | none | Interface for Atlas NDP. Required. | +| `wg_mesh.controller_address` | none | Atlas tenant-0 mesh address on `wg0`. VMs of tenant 0 reach it. | | `traffic_monitor.enabled` | `true` | Enables idle shutdown. | | `migration.final_delta_mib` | `512` | Delta size that triggers the final snapshot. | | `migration.transfer_port` | `9002` | Snapshot stream port. Same on every host. | diff --git a/metal/cmd/metald/config.go b/metal/cmd/metald/config.go index 9b31ca53d..984316160 100644 --- a/metal/cmd/metald/config.go +++ b/metal/cmd/metald/config.go @@ -41,9 +41,10 @@ type migrationOptions struct { // meshOptions configures Atlas WG Mesh. type meshOptions struct { - enabled bool - binaryPath string - uplinkName string + enabled bool + binaryPath string + uplinkName string + controllerAddress string } // trafficMonitorOptions configures VM traffic monitoring. @@ -139,6 +140,7 @@ type wgMeshFile struct { Enabled *bool `toml:"enabled"` BinaryPath string `toml:"binary_path"` Uplink string `toml:"uplink"` + Controller string `toml:"controller_address"` } type trafficFile struct { @@ -184,6 +186,7 @@ func applyFile(resolvedOptions *options, path string) error { overlayBool(&resolvedOptions.mesh.enabled, fc.WGMesh.Enabled) overlay(&resolvedOptions.mesh.binaryPath, fc.WGMesh.BinaryPath) overlay(&resolvedOptions.mesh.uplinkName, fc.WGMesh.Uplink) + overlay(&resolvedOptions.mesh.controllerAddress, fc.WGMesh.Controller) overlayBool(&resolvedOptions.trafficMonitor.enabled, fc.Traffic.Enabled) overlayInt(&resolvedOptions.migration.finalDeltaMiB, fc.Migration.FinalDeltaMiB) overlayInt(&resolvedOptions.migration.transferPort, fc.Migration.TransferPort) diff --git a/metal/cmd/metald/main.go b/metal/cmd/metald/main.go index fdfb1234c..b6a87d6cb 100644 --- a/metal/cmd/metald/main.go +++ b/metal/cmd/metald/main.go @@ -147,6 +147,7 @@ func connectMesh(options options) (*network.Mesh, error) { mesh, err := network.NewMesh(network.MeshConfig{ CommandPath: options.mesh.binaryPath, UplinkName: options.mesh.uplinkName, + ControllerAddress: options.mesh.controllerAddress, WireGuardName: options.wireGuardName, WireGuardStatePath: wireGuardStatePath(options), }) diff --git a/metal/internal/network/mesh.go b/metal/internal/network/mesh.go index 4cf586a29..5439e61bc 100644 --- a/metal/internal/network/mesh.go +++ b/metal/internal/network/mesh.go @@ -37,6 +37,8 @@ type MeshConfig struct { WireGuardName string // WireGuardStatePath holds the managed WireGuard peer state that the mesh reads. WireGuardStatePath string + // ControllerAddress is the Atlas mesh address on wg0. + ControllerAddress string } // Mesh registers virtual machine addresses with the Atlas WG Mesh CLI. @@ -45,6 +47,7 @@ type Mesh struct { uplinkName string wireGuardName string wireGuardStatePath string + controllerAddress string } // NewMesh returns a mesh registrar for one host. @@ -70,6 +73,7 @@ func NewMesh(configuration MeshConfig) (*Mesh, error) { uplinkName: configuration.UplinkName, wireGuardName: configuration.WireGuardName, wireGuardStatePath: configuration.WireGuardStatePath, + controllerAddress: configuration.ControllerAddress, }, nil } @@ -102,7 +106,7 @@ func (mesh *Mesh) PrivateNetworkMAC() (string, error) { // EnsureHost applies the host configuration and refreshes its BPF programs. func (mesh *Mesh) EnsureHost(ctx context.Context) error { return platform.Run(ctx, mesh.commandPath, "configure", - "--uplink", mesh.uplinkName, "--wireguard", mesh.wireGuardName) + "--uplink", mesh.uplinkName, "--wireguard", mesh.wireGuardName, "--controller", mesh.controllerAddress) } // removeVM unregisters one VM address. An address this host does not own is not an error. diff --git a/metal/internal/network/mesh_test.go b/metal/internal/network/mesh_test.go index bc2c8d9b8..4d91e36e5 100644 --- a/metal/internal/network/mesh_test.go +++ b/metal/internal/network/mesh_test.go @@ -105,7 +105,7 @@ func recordingMesh(t *testing.T) (*Mesh, string) { t.Fatal(err) } mesh, err := NewMesh(MeshConfig{ - CommandPath: command, WireGuardName: "wg0", UplinkName: "eno1", + CommandPath: command, WireGuardName: "wg0", UplinkName: "eno1", ControllerAddress: "fdaa:1::ffff:ffff:ffff:ffff", WireGuardStatePath: "/var/lib/metal/wireguard-peers.json", }) if err != nil { @@ -140,7 +140,7 @@ func TestMeshUsesConvergentHostCommands(t *testing.T) { } want := []string{ - "configure --uplink eno1 --wireguard wg0", + "configure --uplink eno1 --wireguard wg0 --controller fdaa:1::ffff:ffff:ffff:ffff", "peers sync /var/lib/metal/wireguard-peers.json --unicast", "privileged-vm replace fdaa:1::1 fdaa:1::2", "privileged-vm clear", diff --git a/scripts/atlas-vm/README.md b/scripts/atlas-vm/README.md index 819279187..b46ad1cf4 100644 --- a/scripts/atlas-vm/README.md +++ b/scripts/atlas-vm/README.md @@ -27,9 +27,15 @@ The setup generates a temporary password for Pilot and the site Administrator. I The setup creates one Secure Shell key for the `pilot.user`. Atlas uses this key to manage Metal Servers. The setup keeps this key when you run it again. -The setup creates the server provider network resources and the Route53 records. It also gets the provider catalogs and a wildcard certificate. Each `[[image]]` table creates one system image in site-file storage. Cargo configures object storage later. +Set `atlas.import_server_id` to import this host as a Metal Server after Atlas setup. The import is optional and continues in the background. Set `atlas.import_storage_pool_device` only if the host needs a different storage device. See [provider server import](../../docs/region/hosts-and-providers.md#import-a-provider-server) for prerequisites and recovery. -The configuration contains provider secrets. `atlas-vm` stores its copy at `/var/lib/atlas-vm/atlas-vm.toml` with mode `0600`. +In Atlas mode, setup creates the Atlas WireGuard peer and its systemd timer. See [Atlas access to hosts](../../docs/region/host-access.md) for the network path and host SSH access. + +Gateway mode runs a development WireGuard gateway instead of Atlas. Deploy it with `pilot --site SITE deploy-dev-gateway `. See [development gateway setup](../../docs/region/host-access.md#development-gateway). + +In Atlas mode, setup creates the server provider network resources and the Route53 records. It also gets the provider catalogs and a wildcard certificate. Each `[[image]]` table creates one system image in site-file storage. Cargo configures object storage later. + +The Atlas mode configuration contains provider secrets. `atlas-vm` stores its copy at `/var/lib/atlas-vm/atlas-vm.toml` with mode `0600`. ## Use the VM @@ -47,7 +53,7 @@ sudo atlas-vm resize --vcpu 8 --disk 60 # This restarts the VM. A disk can o AWS uses one subnet in one availability zone, so a public IPv6 block can move to any host. An AWS region always uses unicast networking, because a VPC does not carry link-local multicast. -The VM answers on port 2222, and host ports 80 and 443 reach it. +The VM answers on host port 2222. In Atlas mode, host ports 80 and 443 also reach it. Gateway mode forwards UDP port 51821 to its WireGuard interface. ## Delete the VM diff --git a/scripts/atlas-vm/atlas-vm.example.toml b/scripts/atlas-vm/atlas-vm.example.toml index 58efcbe16..c64a11e13 100644 --- a/scripts/atlas-vm/atlas-vm.example.toml +++ b/scripts/atlas-vm/atlas-vm.example.toml @@ -28,6 +28,11 @@ private_network_mtu = 1500 # Set this URL when Central sends requests to this Atlas region. central_jwks_url = "" +# Provider server ID of this host, to import it as a Metal Server after setup. +# A storage pool device replaces the provider device. Atlas erases it. +import_server_id = "" +import_storage_pool_device = "" + [atlas.vm_scheduling] use_dedicated_sleepy_vm_hosts = true placement_strategy = "balanced" diff --git a/scripts/atlas-vm/atlas_vm.py b/scripts/atlas-vm/atlas_vm.py index bb6f5c282..3fa6e58b2 100755 --- a/scripts/atlas-vm/atlas_vm.py +++ b/scripts/atlas-vm/atlas_vm.py @@ -34,8 +34,12 @@ "ubuntu-24.04-server-cloudimg-amd64.squashfs" ) ROOTFS_SHA256 = "bb4bc95d539df92c96ad0ed34c017363e4a7a62772c6af1dc3553e06ce710b74" -# The Ubuntu kernel does not boot on the Firecracker device model. Use the CI kernel. -KERNEL_URL = "https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.10/x86_64/vmlinux-5.10.223" +# Firecracker boots only the uncompressed ELF kernel inside the Ubuntu vmlinuz. +KERNEL_URL = ( + "https://cloud-images.ubuntu.com/releases/noble/release-20260518/" + "unpacked/ubuntu-24.04-server-cloudimg-amd64-vmlinuz-generic" +) +KERNEL_SHA256 = "3a33b65c88f98a5563c926d5b163ebe09706e5084ba587a19c1b15bd3e7a82d6" BOOT_ARGUMENTS = "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw" VM_NAME = "atlas" BENCH_NAME = "atlas" @@ -43,6 +47,8 @@ HOST_ADDRESS = "172.16.100.1" VM_ADDRESS = "172.16.100.2" FORWARD_PORTS = (80, 443) +GATEWAY_PORT = 51821 +GATEWAY_DEVELOPER_ADDRESS = "172.16.100.3" SCALEWAY_ZONES = { "fr-par-1", "fr-par-2", @@ -71,6 +77,7 @@ "ssh", "scp", "ssh-keygen", + "zstd", ) SSH_OPTIONS = ( "-o", @@ -130,6 +137,10 @@ class Settings: disk_gib: int = 24 ssh_port: int = 2222 setup_script_url: str = "" + import_server_id: str = "" + import_storage_pool_device: str = "" + private_network_cidr: str = "" + developer_public_key: str = "" @classmethod def read(cls, path: Path) -> Settings: @@ -147,13 +158,17 @@ def read(cls, path: Path) -> Settings: branch = atlas.get("branch", "develop") return cls( path=path, - site=pilot["site"], + site=pilot.get("site", ""), bench_user=pilot.get("user", "frappe"), vcpu_count=int(vm.get("vcpu_count", 4)), memory_mib=int(vm.get("memory_mib", 8192)), disk_gib=int(vm.get("disk_gib", 24)), ssh_port=int(vm.get("ssh_port", 2222)), setup_script_url=f"{raw}/{branch}/scripts/atlas-vm/setup.py", + import_server_id=atlas.get("import_server_id", ""), + import_storage_pool_device=atlas.get("import_storage_pool_device", ""), + private_network_cidr=atlas.get("private_network_cidr", ""), + developer_public_key=document.get("gateway", {}).get("developer_public_key", ""), ) def update_sizes(self, changes: dict[str, int]) -> None: @@ -172,6 +187,12 @@ def update_sizes(self, changes: dict[str, int]) -> None: def validate_configuration(document: dict, path: Path) -> None: """Reject an incomplete deployment configuration before the VM changes.""" + if "gateway" in document: + _validate_keys(document, {"vm", "gateway", "atlas"}, path, "") + _validate_vm_configuration(document.get("vm", {}), path) + _required_string(document["gateway"], "developer_public_key", path, "gateway") + _required_string(_required_table(document, "atlas", path), "private_network_cidr", path, "atlas") + return _validate_keys(document, {"vm", "pilot", "atlas", "image"}, path, "") _validate_vm_configuration(document.get("vm", {}), path) _validate_pilot_configuration(_required_table(document, "pilot", path), path) @@ -214,6 +235,8 @@ def _validate_atlas_configuration(atlas: dict, path: Path) -> None: "private_network_cidr", "private_network_mtu", "central_jwks_url", + "import_server_id", + "import_storage_pool_device", "vm_scheduling", "scaleway", "aws", @@ -228,7 +251,14 @@ def _validate_atlas_configuration(atlas: dict, path: Path) -> None: for key in ("private_network_cidr", "private_network_mtu", "central_jwks_url"): if key not in atlas: raise AtlasVmError(f"{path}: atlas.{key} is required") - for key in ("repository", "branch", "base_url", "central_jwks_url"): + for key in ( + "repository", + "branch", + "base_url", + "central_jwks_url", + "import_server_id", + "import_storage_pool_device", + ): if key in atlas and not isinstance(atlas[key], str): raise AtlasVmError(f"{path}: atlas.{key} must be a string") provider = atlas["server_provider"] @@ -426,7 +456,7 @@ def generate_password(length: int = 24) -> str: def find_host_key() -> Path: - """The VM trusts the key this host already uses for Secure Shell.""" + """The VM trusts the key this host already uses for Secure Shell. A new host gets a root key.""" homes = [Path("/root")] if os.environ.get("SUDO_USER"): homes.append(Path("/home") / os.environ["SUDO_USER"]) @@ -435,7 +465,10 @@ def find_host_key() -> Path: candidate = home / ".ssh" / name if candidate.is_file() and candidate.with_suffix(".pub").is_file(): return candidate - raise AtlasVmError("no Secure Shell key pair for this host; create one with: ssh-keygen -t ed25519") + key = Path("/root/.ssh/id_ed25519") + key.parent.mkdir(mode=0o700, exist_ok=True) + run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)]) + return key class GuestImage: @@ -518,7 +551,7 @@ def firecracker_binary(self) -> Path: @property def kernel_image(self) -> Path: - return self.downloads / "vmlinux" + return self.downloads / "vmlinux-6.8.0-117" @property def vm_directory(self) -> Path: @@ -571,6 +604,25 @@ def download(url: str, path: Path, checksum: str = "") -> None: detail(f"{size / 1024**2:.0f} MiB" if size >= 1024**2 else f"{size / 1024:.0f} KiB") +def extract_kernel(vmlinuz: Path, path: Path) -> None: + if path.exists(): + return + data = vmlinuz.read_bytes() + staged = path.with_suffix(".part") + with staged.open("wb") as output: + # zstd rejects the trailing bzImage data after it writes the kernel. + subprocess.run( + ["zstd", "-cdq"], + input=data[data.find(b"\x28\xb5\x2f\xfd") :], + stdout=output, + stderr=subprocess.DEVNULL, + ) + if staged.read_bytes()[:4] != b"\x7fELF": + staged.unlink() + raise AtlasVmError(f"{vmlinuz.name} holds no ELF kernel") + staged.rename(path) + + class ConsoleReader: """Follow the guest console. The kept lines explain a boot that never reaches Secure Shell.""" @@ -602,6 +654,36 @@ def report(self) -> None: guest(line) +GATEWAY_SCRIPT = r"""set -eu +interface=atlas-gateway +# The guest root file system has no modules for its kernel. WireGuard needs them. +packages="wireguard-tools linux-modules-$(uname -r)" +if ! dpkg -s $packages >/dev/null 2>&1; then + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $packages >/dev/null +fi +install -d -m 700 /etc/wireguard +[ -f /etc/wireguard/$interface.key ] || (umask 077 && wg genkey > /etc/wireguard/$interface.key) +config="[Interface] +ListenPort = $LISTEN_PORT +PostUp = wg set %i private-key /etc/wireguard/$interface.key + +[Peer] +PublicKey = $DEVELOPER_PUBLIC_KEY +AllowedIPs = $DEVELOPER_ADDRESS/32" +if [ "$(cat /etc/wireguard/$interface.conf 2>/dev/null || true)" != "$config" ]; then + (umask 077 && printf '%s\n' "$config" > /etc/wireguard/$interface.conf) + systemctl restart wg-quick@$interface +fi +systemctl enable --quiet wg-quick@$interface +systemctl is-active --quiet wg-quick@$interface || systemctl restart wg-quick@$interface +# The developer address is in the guest subnet, so the host masquerade carries it. +printf 'net.ipv4.ip_forward = 1\nnet.ipv4.conf.eth0.proxy_arp = 1\n' > /etc/sysctl.d/99-atlas-gateway.conf +sysctl -q -p /etc/sysctl.d/99-atlas-gateway.conf +wg show $interface public-key +""" + + NETWORK_SCRIPT_BODY = r""" # Each forward matches the host address, because a match on the port alone also # captures traffic that another guest sends through this host to a remote server. @@ -613,9 +695,16 @@ def report(self) -> None: [[ -n $uplink_address ]] || { echo "$uplink has no IPv4 address" >&2; exit 1; } echo "-t nat -A POSTROUTING -s $vm_address/32 -o $uplink -j MASQUERADE" + # The guest subnet reaches the provider private network behind this host. + if [[ -n $private_network_cidr ]]; then + echo "-t nat -I POSTROUTING -s ${host_address%.*}.0/24 -d $private_network_cidr -j MASQUERADE" + fi + for pair in $udp_forwards; do + echo "-t nat -A PREROUTING -d $uplink_address -p udp --dport ${pair%%:*} -j DNAT --to-destination $vm_address:${pair##*:}" + done echo "-t nat -A POSTROUTING -s 127.0.0.0/8 -d $vm_address -j SNAT --to-source $host_address" - echo "-I FORWARD -i $tap_device -j ACCEPT" - echo "-I FORWARD -o $tap_device -j ACCEPT" + echo "-t filter -I FORWARD -i $tap_device -j ACCEPT" + echo "-t filter -I FORWARD -o $tap_device -j ACCEPT" for pair in $forwards; do host_port=${pair%%:*} guest_port=${pair##*:} @@ -689,7 +778,8 @@ def is_created(self) -> bool: @property def port_forwards(self) -> list[tuple[int, int]]: - return [(self.settings.ssh_port, 22)] + [(port, port) for port in FORWARD_PORTS] + ports = () if self.settings.developer_public_key else FORWARD_PORTS + return [(self.settings.ssh_port, 22)] + [(port, port) for port in ports] def ssh_arguments(self, command: list[str] | None = None) -> list[str]: arguments = [ @@ -730,6 +820,7 @@ def run_in_guest(self, command: str) -> int: def write_network_script(self) -> None: forwards = " ".join(f"{host}:{guest}" for host, guest in self.port_forwards) + cidr = self.settings.private_network_cidr header = f"""#!/usr/bin/env bash # Host network for the {VM_NAME} VM. Generated by atlas-vm. set -euo pipefail @@ -738,6 +829,8 @@ def write_network_script(self) -> None: host_address={HOST_ADDRESS} vm_address={VM_ADDRESS} forwards="{forwards}" +udp_forwards="{f"{GATEWAY_PORT}:{GATEWAY_PORT}" if self.settings.developer_public_key else ""}" +private_network_cidr="{cidr}" """ write_file(self.paths.network_script, header + NETWORK_SCRIPT_BODY, mode=0o755) @@ -779,6 +872,7 @@ def write_unit(self) -> None: [Service] Type=exec +ExecStartPre=/bin/rm -f {self.paths.api_socket} ExecStartPre={self.paths.network_script} start ExecStart={self.paths.firecracker_binary} --api-sock {self.paths.api_socket} --config-file {self.paths.configuration} ExecStopPost={self.paths.network_script} stop @@ -792,6 +886,7 @@ def write_unit(self) -> None: """, ) run(["systemctl", "daemon-reload"]) + run(["systemctl", "enable", "--quiet", SERVICE_NAME]) def install_firecracker(self) -> None: if self.paths.firecracker_binary.exists(): @@ -888,6 +983,42 @@ def run_setup(self, script: Path | None = None) -> None: if process.returncode != 0: raise AtlasVmError(f"setup.py failed; read {self.paths.setup_log}") + def setup_gateway(self) -> None: + """Relay the developer WireGuard link to the provider private network.""" + step("set up the development gateway") + environment = f"DEVELOPER_PUBLIC_KEY={shlex.quote(self.settings.developer_public_key)} LISTEN_PORT={GATEWAY_PORT} DEVELOPER_ADDRESS={GATEWAY_DEVELOPER_ADDRESS}" + result = subprocess.run( + self.ssh_arguments([f"{environment} bash -s"]), + input=GATEWAY_SCRIPT, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise AtlasVmError(f"gateway setup failed: {result.stdout[-500:]}{result.stderr[-500:]}") + print(f"gateway public key: {result.stdout.strip().splitlines()[-1]}") + + def import_host(self) -> None: + """Add this host to Atlas as a Metal Server. Atlas connects to it with the key of its bench user.""" + settings = self.settings + if not settings.import_server_id: + return + + step(f"import this host as Metal Server {settings.import_server_id}") + result = subprocess.run( + self.ssh_arguments([f"ssh-keygen -y -f /home/{settings.bench_user}/.ssh/id_ed25519"]), + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise AtlasVmError(f"could not read the Atlas SSH key in the VM: {result.stderr.strip()}") + authorize_root_key(result.stdout.strip()) + + command = f"pilot frappe --site {settings.site} import-metal-server {shlex.quote(settings.import_server_id)}" + if settings.import_storage_pool_device: + command += f" --storage-pool-device {shlex.quote(settings.import_storage_pool_device)}" + if self.run_as_bench(f"{command} --bench {BENCH_NAME}") != 0: + raise AtlasVmError("Atlas could not import this host") + DESTROY_WARNING = """DANGER: this deletes the VM in {directory}. The bench, every site, every database, and every file in the guest disk are gone for ever. There is no @@ -918,6 +1049,17 @@ def require_host_support() -> None: raise AtlasVmError("this CLI supports x86_64 only") +def authorize_root_key(public_key: str) -> None: + """Let one key log in as root. The host firewall later limits SSH to wg0 and the private network.""" + ssh_directory = Path("/root/.ssh") + ssh_directory.mkdir(mode=0o700, exist_ok=True) + authorized_keys = ssh_directory / "authorized_keys" + lines = authorized_keys.read_text().splitlines() if authorized_keys.exists() else [] + if public_key not in lines: + authorized_keys.write_text("\n".join([*lines, public_key]) + "\n") + authorized_keys.chmod(0o600) + + def install_self() -> None: source = Path(__file__).resolve() if source == INSTALLED_PATH: @@ -949,7 +1091,8 @@ def command_create(machine: VirtualMachine, arguments: argparse.Namespace) -> No warn(f"{paths.base} has {free_gib}G free for a {machine.settings.disk_gib}G guest disk") machine.install_firecracker() - download(KERNEL_URL, paths.kernel_image) + download(KERNEL_URL, paths.downloads / "vmlinuz", KERNEL_SHA256) + extract_kernel(paths.downloads / "vmlinuz", paths.kernel_image) if arguments.rebuild and paths.rootfs_image.exists(): print(DESTROY_WARNING.format(directory=paths.vm_directory), file=sys.stderr) @@ -969,8 +1112,11 @@ def command_create(machine: VirtualMachine, arguments: argparse.Namespace) -> No install_self() machine.start(arguments.verbose) - if not arguments.skip_setup: + if machine.settings.developer_public_key: + machine.setup_gateway() + elif not arguments.skip_setup: machine.run_setup(arguments.script) + machine.import_host() command_status(machine, arguments) @@ -1041,7 +1187,11 @@ def command_logs(machine: VirtualMachine, arguments: argparse.Namespace) -> None def command_setup(machine: VirtualMachine, arguments: argparse.Namespace) -> None: if not machine.is_running: raise AtlasVmError(f"{SERVICE_NAME} is not running; start it with: atlas-vm start") + if machine.settings.developer_public_key: + machine.setup_gateway() + return machine.run_setup(arguments.script) + machine.import_host() def command_reset_password(machine: VirtualMachine, arguments: argparse.Namespace) -> None: diff --git a/scripts/atlas-vm/setup.py b/scripts/atlas-vm/setup.py index 6412a54e8..f0bdfe6e7 100755 --- a/scripts/atlas-vm/setup.py +++ b/scripts/atlas-vm/setup.py @@ -234,6 +234,9 @@ def install_packages(self) -> None: "squashfs-tools", "zstd", "e2fsprogs", + "wireguard-tools", + # The guest root file system has no modules for the Atlas VM kernel. nft and WireGuard need them. + f"linux-modules-{os.uname().release}", ], env=environment, ) @@ -355,9 +358,21 @@ def configure_atlas(self) -> None: input_text=json.dumps(values), ) + def configure_wireguard(self) -> None: + configuration = self.configuration + step("stage 12: Atlas WireGuard peer") + site = configuration.site + self.pilot(f"frappe --site {site} configure-atlas-wireguard") + run( + [ + str(configuration.bench_path / "apps/atlas/scripts/install-atlas-wireguard.sh"), + str(configuration.bench_path / "sites" / site / "private/wireguard/atlas0.conf"), + ] + ) + def grant_image_builder_sudo(self) -> None: # The image builder runs its root file system build through sudo on every build. - step("stage 12: sudo grant for the image builder") + step("stage 13: sudo grant for the image builder") self.install_grant( self.image_builder_grant, f"{self.configuration.bench_user} ALL=(ALL) NOPASSWD: {self.configuration.image_builder_path} *", @@ -366,7 +381,7 @@ def grant_image_builder_sudo(self) -> None: def build_images(self) -> None: # Bootstrap has no object storage credentials yet, so every image is a site file. configuration = self.configuration - step(f"stage 13: guest images ({len(configuration.images)})") + step(f"stage 14: guest images ({len(configuration.images)})") for version, architecture, minimal in configuration.images: step(f"image {version} {architecture} {'minimal' if minimal else 'server'}") arguments = ( @@ -389,6 +404,7 @@ def run(self) -> None: self.setup_production() self.install_atlas() self.configure_atlas() + self.configure_wireguard() finally: Path(self.setup_grant).unlink(missing_ok=True) self.grant_image_builder_sudo() diff --git a/scripts/atlas-vm/test_configuration.py b/scripts/atlas-vm/test_configuration.py index 971b7fc16..c5ec27196 100644 --- a/scripts/atlas-vm/test_configuration.py +++ b/scripts/atlas-vm/test_configuration.py @@ -43,6 +43,20 @@ def setUp(self) -> None: self.path = Path(self.temporary_directory.name) / "atlas-vm.toml" self.path.write_text(EXAMPLE.read_text()) + def test_the_host_import_values_are_read(self) -> None: + text = self.path.read_text().replace('import_server_id = ""', 'import_server_id = "server-1"') + self.path.write_text( + text.replace( + 'import_storage_pool_device = ""', 'import_storage_pool_device = "/root/disks/atlas.img"' + ) + ) + + host = atlas_vm.Settings.read(self.path) + + self.assertEqual( + (host.import_server_id, host.import_storage_pool_device), ("server-1", "/root/disks/atlas.img") + ) + def test_example_is_valid_for_both_readers(self) -> None: host = atlas_vm.Settings.read(self.path) with patch.object(setup, "generate_password", return_value="generated-bootstrap-password"): @@ -252,6 +266,8 @@ def setUp(self) -> None: f"host_address={atlas_vm.HOST_ADDRESS}\n" f"vm_address={atlas_vm.VM_ADDRESS}\n" 'forwards="443:443"\n' + 'udp_forwards="51821:51821"\n' + 'private_network_cidr="10.1.0.0/20"\n' + atlas_vm.NETWORK_SCRIPT_BODY.replace('case "${1:-}" in', 'rules\nexit 0\ncase "${1:-}" in') ) script.chmod(0o755) @@ -268,7 +284,7 @@ def test_a_forward_matches_only_the_host_address(self) -> None: f"-t nat -A PREROUTING -d {self.uplink_address} -p tcp --dport 443 " f"-j DNAT --to-destination {atlas_vm.VM_ADDRESS}:443" ) - prerouting = [rule for rule in self.rules if "-A PREROUTING" in rule] + prerouting = [rule for rule in self.rules if "-A PREROUTING" in rule and "-p tcp" in rule] self.assertEqual(prerouting, [expected]) def test_the_host_reaches_a_forward_through_its_own_address(self) -> None: diff --git a/scripts/install-atlas-wireguard.sh b/scripts/install-atlas-wireguard.sh new file mode 100755 index 000000000..b593679f7 --- /dev/null +++ b/scripts/install-atlas-wireguard.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Apply one Atlas wg-quick file now and within 10 seconds of each Atlas rewrite. Run as root on the Atlas machine. + +set -eu + +config_file=${1:?usage: install-atlas-wireguard.sh /path/to/sites/SITE/private/wireguard/atlas0.conf} +interface=$(basename "$config_file" .conf) +unit=atlas-wireguard-$interface +# Tenant-0 VMs reach Atlas through this interface. WireGuard itself needs no port here. +tcp_ports=${ATLAS_WIREGUARD_TCP_PORTS:-22, 80, 443, 2222} +rules_file=/etc/atlas/$unit.nft + +if [ "$(id -u)" -ne 0 ]; then + echo "install-atlas-wireguard must run as root" >&2 + exit 1 +fi +if [ ! -f "$config_file" ]; then + echo "$config_file does not exist. Run pilot --site SITE configure-atlas-wireguard first." >&2 + exit 1 +fi +if ! command -v nft >/dev/null; then + echo "install-atlas-wireguard needs nft. Install nftables first." >&2 + exit 1 +fi + +install -d -m 0755 /etc/atlas +cat > "$rules_file" < /etc/systemd/system/$unit.service </dev/null 2>&1 && exit 0; install -m 0600 $config_file /etc/wireguard/$interface.conf; if ip link show $interface >/dev/null 2>&1; then wg syncconf $interface <(wg-quick strip $interface); else wg-quick up $interface; fi' +EOF + +# SELinux can hide a home directory from a path unit, so a timer polls the file. +cat > /etc/systemd/system/$unit.timer <expires_ns > bpf_ktime_get_ns() ? &moved->virtual_machine : NULL; } +/* The gateway for this VM's traffic to the destination, or NULL. The longest destination prefix wins. */ static __always_inline struct in6_addr *get_gateway_route(const struct in6_addr *source, const struct in6_addr *destination) { struct route_key key = { @@ -172,6 +189,7 @@ static __always_inline int has_gateway_return_route(const struct in6_addr *virtu return get_gateway_route(virtual_machine, foreign_source) != NULL; } +/* Whether the interface belongs to a local gateway VM. */ static __always_inline int is_gateway_interface(__u32 ifindex) { return bpf_map_lookup_elem(&gateways, &ifindex) != NULL; @@ -185,6 +203,7 @@ static __always_inline __u32 get_gateway_interface(const struct in6_addr *addres return ifindex && is_gateway_interface(*ifindex) ? *ifindex : 0; } +/* The host peer with this uplink IPv4 address. The list ends at the first empty entry. */ static __always_inline struct peer *find_peer_by_ipv4(__be32 ipv4) { for (__u32 index = 0; index < PEER_LIMIT; index++) @@ -201,6 +220,7 @@ static __always_inline struct peer *find_peer_by_ipv4(__be32 ipv4) return NULL; } +/* The host peer with this uplink MAC address. The list ends at the first empty entry. */ static __always_inline struct peer *find_peer_by_mac(const __u8 *mac) { __u64 packed = pack_mac(mac); @@ -235,6 +255,7 @@ static __always_inline int take_discovery_token(__u32 ifindex) return 1; } +/* Allow one unsolicited advertisement for each moved address in each interval. */ static __always_inline int take_announcement_token(const struct in6_addr *address) { __u64 now = bpf_ktime_get_ns(); diff --git a/services/wg-mesh/bpf/vm.h b/services/wg-mesh/bpf/vm.h index 9564a9248..ff369be2f 100644 --- a/services/wg-mesh/bpf/vm.h +++ b/services/wg-mesh/bpf/vm.h @@ -134,6 +134,9 @@ int handle_vm_packet(struct __sk_buff *packet) if (!source_is_allowed(packet->ifindex, &source, &destination)) return TC_ACT_SHOT; + if (is_controller(&destination)) + return is_vm_address(&source) ? TC_ACT_OK : TC_ACT_SHOT; + /* Linux delivers same-host traffic through the VM host route. */ if (is_local_vm(&destination)) return is_vm_address(&source) || has_gateway_return_route(&destination, &source) ? TC_ACT_OK : TC_ACT_SHOT; diff --git a/services/wg-mesh/cli/host.go b/services/wg-mesh/cli/host.go index 42c419a35..d56254fc1 100644 --- a/services/wg-mesh/cli/host.go +++ b/services/wg-mesh/cli/host.go @@ -16,7 +16,7 @@ import ( const vmLockPath = "/run/lock/atlas-wg-mesh.lock" -var uplinkName, wireGuardName string +var uplinkName, wireGuardName, controllerText string var resetForce bool var statusJSON bool @@ -25,8 +25,16 @@ var configureCommand = &cobra.Command{ Use: "configure", Short: "install or refresh Atlas WG Mesh on this host", Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { - return configureHost(uplinkName, wireGuardName) + RunE: func(command *cobra.Command, _ []string) error { + var controller *[16]byte + if command.Flags().Changed("controller") { + address, err := parseController(controllerText) + if err != nil { + return err + } + controller = &address + } + return configureHost(uplinkName, wireGuardName, controller) }, } @@ -60,6 +68,7 @@ var versionCommand = &cobra.Command{ func init() { configureCommand.Flags().StringVar(&uplinkName, "uplink", "", "mesh uplink interface") configureCommand.Flags().StringVar(&wireGuardName, "wireguard", "", "WireGuard interface") + configureCommand.Flags().StringVar(&controllerText, "controller", "", "Atlas tenant-0 mesh address on wg0") configureCommand.MarkFlagRequired("uplink") configureCommand.MarkFlagRequired("wireguard") resetCommand.Flags().BoolVar(&resetForce, "force", false, "remove mesh state while local VMs remain") @@ -68,8 +77,23 @@ func init() { rootCommand.AddCommand(configureCommand, resetCommand, statusCommand, versionCommand) } -// configureHost installs the mesh or replaces its BPF without detaching a live hook. -func configureHost(uplinkName, wireGuardName string) error { +// parseController accepts an empty value or a tenant-0 VM address. +func parseController(text string) ([16]byte, error) { + if text == "" { + return [16]byte{}, nil + } + address, err := parseMeshAddress(text) + if err != nil { + return [16]byte{}, err + } + if address[4]|address[5]|address[6]|address[7] != 0 { + return [16]byte{}, fmt.Errorf("controller %s is not a tenant-0 address", text) + } + return address, nil +} + +// configureHost installs the mesh or replaces its BPF without detaching a live hook. A nil controller keeps the stored address. +func configureHost(uplinkName, wireGuardName string, controller *[16]byte) error { unlock, err := lockFile(vmLockPath, true) if err != nil { return err @@ -122,6 +146,11 @@ func configureHost(uplinkName, wireGuardName string) error { candidate.cleanup() return errors.Join(err, rollbackError) } + if controller != nil { + if err := writeMap("controller_address", uint32(0), *controller); err != nil { + return fmt.Errorf("write the controller address: %w", err) + } + } fmt.Printf("Atlas WG Mesh runs on %s and %s\n", uplinkName, wireGuardName) return errors.Join(removeOldReleases(), removeObsoleteMaps(candidate.maps)) diff --git a/services/wg-mesh/cli/vm_test.go b/services/wg-mesh/cli/vm_test.go index 36dba345f..14fcbc7e2 100644 --- a/services/wg-mesh/cli/vm_test.go +++ b/services/wg-mesh/cli/vm_test.go @@ -34,3 +34,17 @@ func TestParseVMStateRejectsBadInput(t *testing.T) { } } } + +func TestParseControllerAcceptsOnlyATenantZeroMeshAddress(t *testing.T) { + if address, err := parseController(""); err != nil || address != [16]byte{} { + t.Fatalf("an empty controller must clear the map, got %v, %v", address, err) + } + if _, err := parseController("fdaa:1::ffff:ffff:ffff:ffff"); err != nil { + t.Fatal(err) + } + for _, text := range []string{"fdab:1::7", "fdaa:1:0:2::1", "10.1.0.2"} { + if _, err := parseController(text); err == nil { + t.Errorf("accepted controller %q", text) + } + } +}