diff --git a/app/catalog.py b/app/catalog.py index 692c2f0f..16075441 100644 --- a/app/catalog.py +++ b/app/catalog.py @@ -153,6 +153,8 @@ def _validate(data: dict[str, Any], path: Path) -> list[str]: re.compile(pattern) except (re.error, TypeError): errors.append(f"{path.name}: docker.env[{i}].pattern must be a valid regular expression") + if isinstance(item, dict) and "per_device" in item and not isinstance(item["per_device"], bool): + errors.append(f"{path.name}: docker.env[{i}].per_device must be a boolean") signals = docker.get("health_signals") if isinstance(signals, list): for i, item in enumerate(signals): @@ -222,6 +224,9 @@ def _validate(data: dict[str, Any], path: Path) -> list[str]: url = str(link.get("url") or "") if isinstance(link, dict) else "" if not isinstance(link, dict) or link.get("id_env") not in public: errors.append(f"{path.name}: device_link.id_env must name a non-secret docker.env key") + elif link["id_env"] not in {v.get("key") for v in declared if v.get("per_device") is True}: + # Copies through a proxy derive their own id only from marked keys. + errors.append(f"{path.name}: device_link.id_env must be marked per_device: true") elif not isinstance(link.get("steps"), str) or not link["steps"].strip(): errors.append(f"{path.name}: device_link.steps must say what to do with the id") elif url and not url.startswith("https://"): diff --git a/app/main.py b/app/main.py index 218a1962..ee6e5e70 100644 --- a/app/main.py +++ b/app/main.py @@ -4249,7 +4249,10 @@ async def api_add_proxied_copy( detail=f"Deploy {svc.get('name') or slug} on this worker first; the proxied copy runs with its settings.", ) docker_conf = svc.get("docker") or {} - earner = proxies.earner_spec(recorded, docker_conf, proxy_id) + try: + earner = proxies.earner_spec(recorded, docker_conf, proxy_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=f"{svc.get('name') or slug}: {exc}") from exc if not recorded.get("custom_image") and isinstance(docker_conf.get("image_by_arch"), dict): worker = await database.get_worker(worker_id) or {} arch_image = _image_for_arch(docker_conf, _safe_json(worker.get("system_info", "{}"), {}).get("arch")) @@ -4301,6 +4304,10 @@ async def api_add_proxied_copy( if limit == 1 else "Copies on one IP share its bandwidth, and the provider may limit devices per IP." ) + # The copy is a new device: add it to the account as a deploy does, or show the step. + if step := catalog.device_link_step(svc, earner["env"]): + step["registered"] = await _register_device(slug, step["id"], earner["env"]) + response["device_link"] = step return response diff --git a/app/proxies.py b/app/proxies.py index 0f8dbc07..77a67d68 100644 --- a/app/proxies.py +++ b/app/proxies.py @@ -12,6 +12,7 @@ from __future__ import annotations +import hashlib import re from datetime import UTC, date, datetime from typing import Any @@ -216,12 +217,35 @@ def refusal(svc: dict[str, Any]) -> str | None: return None +def copy_device_id(device_id: str, proxy_id: int, pattern: str | None = None) -> str: + """The device id of the copy through proxy ``proxy_id``: same format, its own value. + + The provider knows each device by this id (ProxyRack's 64-hex UUID), so a + copy that kept the original's would report as the same device and earn + nothing of its own. Derived, not random, so a redeploy of the copy keeps + its device. The derived id keeps the original's length and case, and must + still match the catalog's ``pattern`` when there is one. Any other format + raises ValueError: a copy is refused rather than given a cloned identity. + """ + if not re.fullmatch(r"[0-9a-f]+|[0-9A-F]+", device_id): + raise ValueError("its device id is not in a format a copy can derive its own from") + digest = hashlib.shake_256(f"{device_id}-p{proxy_id}".encode()).hexdigest((len(device_id) + 1) // 2) + digest = digest[: len(device_id)] + new_id = digest.upper() if device_id.isupper() else digest + if pattern and not re.fullmatch(pattern, new_id): + raise ValueError("the device id derived for the copy does not match the catalog's format for it") + return new_id + + def earner_spec(recorded: dict[str, Any], docker_conf: dict[str, Any], proxy_id: int) -> dict[str, Any]: """The spec of a proxied copy: the deployed copy's, under its own device names. Built from what the service was actually deployed with on that worker (the record api_deploy writes after its own validation), the same way main._instance_spec makes extra copies, so the two paths cannot drift. + A device id the catalog marks ``per_device`` gets a value of its own too, + wherever it appears (a device name filled from it included). Raises + ValueError when one cannot be derived. """ suffix = f"p{proxy_id}" named = {var["key"] for var in docker_conf.get("env", []) if "{hostname}" in str(var.get("default", ""))} @@ -229,6 +253,23 @@ def earner_spec(recorded: dict[str, Any], docker_conf: dict[str, Any], proxy_id: key: (f"{value}-{suffix}" if key in named and value else value) for key, value in (recorded.get("env") or {}).items() } + renamed: dict[str, str] = {} + for var in docker_conf.get("env", []): + old_id = str(env.get(var.get("key")) or "") if var.get("per_device") else "" + if var.get("per_device") and var.get("required") and not old_id: + # Empty is only safe where the service makes its own id when given none. + raise ValueError(f"its deployment has no {var.get('key')}, so the copy cannot be given one of its own") + if old_id: + renamed[old_id] = copy_device_id(old_id, proxy_id, var.get("pattern")) + + def rename(value: Any) -> Any: + if not isinstance(value, str): + return value + for old_id, new_id in renamed.items(): + value = value.replace(old_id, new_id) + return value + + env = {key: rename(value) for key, value in env.items()} spec: dict[str, Any] = { "image": recorded.get("custom_image") or recorded.get("image"), "env": env, @@ -240,7 +281,7 @@ def earner_spec(recorded: dict[str, Any], docker_conf: dict[str, Any], proxy_id: if recorded.get("command") and docker_conf.get("command"): spec["command"] = re.sub(r"\$\{(\w+)\}", lambda m: env.get(m.group(1), m.group(0)), docker_conf["command"]) elif recorded.get("command"): - spec["command"] = recorded["command"] + spec["command"] = rename(recorded["command"]) return spec diff --git a/app/templates/fleet.html b/app/templates/fleet.html index 94d12513..a043ee19 100644 --- a/app/templates/fleet.html +++ b/app/templates/fleet.html @@ -1061,6 +1061,9 @@

Proxies

const out = await CP.api(`/api/proxies/${proxyId}/copies`, {method: 'POST', body}); CP.toast(`Running through ${out.exit_ip || 'the proxy'}${out.country ? ' ยท ' + out.country : ''}`, 'success'); if (out.warning) CP.toast(out.warning, 'warning'); + // The copy is a new device: say so when it could not be added to the account. + const link = out.device_link; + if (link && !link.registered) CP.toast(`New device ${link.id}: ${link.steps}${link.url ? ' ' + link.url : ''}`, 'warning'); load(); } catch (err) { CP.toast(err.message, 'error'); } }); diff --git a/services/_schema.yml b/services/_schema.yml index 908a53cd..cec8eae8 100644 --- a/services/_schema.yml +++ b/services/_schema.yml @@ -74,6 +74,10 @@ # pattern: "" (optional) regex a typed value must match in full, or the # deploy is refused. For values that land on a command line, where a # stray space or word becomes an argument and the service stops starting. +# per_device: false (optional) the value is the id the provider knows this +# device by (ProxyRack's UUID). A proxied copy gets its own, derived in the +# same format; a value in a format it cannot derive refuses the copy. +# Required on the key a device_link names. # port_range: false (optional) the value is "start:end" with # 1 <= start <= end <= 65535. A pattern checks the shape, not the order. # ports: ["port:port"] diff --git a/services/bandwidth/earnapp.yml b/services/bandwidth/earnapp.yml index 7a6d9f7c..f613e0f5 100644 --- a/services/bandwidth/earnapp.yml +++ b/services/bandwidth/earnapp.yml @@ -29,6 +29,7 @@ docker: oom_score_adj: 300 env: - key: EARNAPP_UUID + per_device: true label: "Node UUID" required: true secret: false diff --git a/services/bandwidth/proxyrack.yml b/services/bandwidth/proxyrack.yml index 6aa6cb5a..c3723cf8 100644 --- a/services/bandwidth/proxyrack.yml +++ b/services/bandwidth/proxyrack.yml @@ -23,6 +23,7 @@ docker: oom_score_adj: 300 env: - key: UUID + per_device: true label: "Device UUID" required: true secret: false diff --git a/services/bandwidth/speedshare.yml b/services/bandwidth/speedshare.yml index a2301689..fb9eaf11 100644 --- a/services/bandwidth/speedshare.yml +++ b/services/bandwidth/speedshare.yml @@ -25,6 +25,7 @@ docker: secret: true description: "Your SpeedShare authorization code from the dashboard" - key: SPEEDSHARE_UUID + per_device: true label: "Device UUID" required: false secret: false diff --git a/services/bandwidth/spide.yml b/services/bandwidth/spide.yml index 6a0b7f6e..3bc9b11f 100644 --- a/services/bandwidth/spide.yml +++ b/services/bandwidth/spide.yml @@ -21,6 +21,7 @@ docker: platforms: [] env: - key: SPIDE_MACHINE_ID + per_device: true label: "Machine ID" required: false secret: false diff --git a/tests/test_device_link_step.py b/tests/test_device_link_step.py index 2e1aec01..bd79f6ba 100644 --- a/tests/test_device_link_step.py +++ b/tests/test_device_link_step.py @@ -106,7 +106,8 @@ def _errors(tmp_path: Path, link) -> list[str]: "docker": { "image": "x/x:1", "env": [ - {"key": "ID"}, + {"key": "ID", "per_device": True}, + {"key": "PLAIN"}, {"key": "TOKEN", "secret": True}, {"key": "TWICE"}, {"key": "TWICE", "secret": True}, @@ -131,6 +132,7 @@ def test_the_loader_accepts_a_well_formed_block(tmp_path): {"id_env": "ID", "url": "https://x.test/${TOKEN}", "steps": "Open it."}, # a secret in the link {"id_env": "ID", "url": "https://x.test/${NOPE}", "steps": "Open it."}, # an undeclared placeholder {"id_env": "TWICE", "steps": "Open it."}, # declared twice, once secret: secret + {"id_env": "PLAIN", "steps": "Open it."}, # not marked per_device, so a proxied copy would clone it "EARNAPP_UUID", # not a mapping ], ) @@ -138,6 +140,18 @@ def test_the_loader_refuses_a_malformed_block(tmp_path, link): assert any("device_link" in e for e in _errors(tmp_path, link)) +def test_per_device_must_be_a_boolean(tmp_path): + data = { + "name": "X", + "slug": "x", + "category": "bandwidth", + "status": "active", + "description": "x", + "docker": {"image": "x/x:1", "env": [{"key": "ID", "per_device": "yes"}]}, + } + assert any("per_device must be a boolean" in e for e in catalog._validate(data, tmp_path / "x.yml")) + + # ProxyRack's peer API adds a device itself (POST /api/device/add), so with the # collector's API key saved the deploy registers the UUID instead of asking the # user to. Answers below are the shapes the API returned on 2026-10-29. diff --git a/tests/test_proxies.py b/tests/test_proxies.py index e64cc0ce..f84f3e2b 100644 --- a/tests/test_proxies.py +++ b/tests/test_proxies.py @@ -4,6 +4,7 @@ import asyncio import json +import re from datetime import date from unittest.mock import AsyncMock, MagicMock, patch @@ -124,6 +125,61 @@ def test_the_copy_has_its_own_device_names_and_nothing_of_the_host(self): assert spec["command"] == "run --device cashpilot-box-p4 --token t0k" assert not {"ports", "volumes", "network_mode", "hostname", "instances"} & set(spec) + def test_the_copy_reports_its_own_device_id_not_the_original(self): + original = "AB" * 32 # ProxyRack's own generator writes 64 uppercase hex + docker_conf = { + "env": [{"key": "UUID", "per_device": True}, {"key": "DEVICE_NAME", "default": "Device-${UUID}"}], + "command": "run ${UUID}", + } + recorded = { + "image": "x/x:1", + "env": {"UUID": original, "DEVICE_NAME": f"Device-{original}", "API_KEY": "k"}, + "command": f"run {original}", + } + spec = proxies.earner_spec(recorded, docker_conf, 1) + new = spec["env"]["UUID"] + assert new != original and re.fullmatch(r"[0-9A-F]{64}", new) + assert spec["env"]["DEVICE_NAME"] == f"Device-{new}" and spec["env"]["API_KEY"] == "k" + assert spec["command"] == f"run {new}" + # Derived, so a redeploy keeps the device; another proxy gets another one. + assert proxies.earner_spec(recorded, docker_conf, 1)["env"]["UUID"] == new + assert proxies.earner_spec(recorded, docker_conf, 2)["env"]["UUID"] not in (new, original) + + def test_an_unmarked_id_is_left_alone(self): + recorded = {"env": {"UUID": "ab" * 32}} + assert proxies.earner_spec(recorded, {"env": [{"key": "UUID"}]}, 1)["env"]["UUID"] == "ab" * 32 + + @pytest.mark.parametrize("value", ["sdk-node-0123abcd", "AbCdEf01", "not hex"]) + def test_a_device_id_in_a_format_it_cannot_derive_refuses_the_copy(self, value): + with pytest.raises(ValueError): + proxies.earner_spec({"env": {"ID": value}}, {"env": [{"key": "ID", "per_device": True}]}, 1) + + def test_a_required_device_id_missing_from_the_deployment_refuses_the_copy(self): + conf = {"env": [{"key": "UUID", "per_device": True, "required": True}]} + with pytest.raises(ValueError): + proxies.earner_spec({"env": {}}, conf, 1) + # Optional and empty: the service makes its own id, nothing is cloned. + optional = {"env": [{"key": "UUID", "per_device": True}]} + assert proxies.earner_spec({"env": {"UUID": ""}}, optional, 1)["env"] == {"UUID": ""} + + def test_a_derived_id_that_misses_the_catalog_pattern_refuses_the_copy(self): + with pytest.raises(ValueError): + proxies.copy_device_id("ab" * 32, 1, pattern=r"[0-9]{64}") + assert re.fullmatch(r"[0-9a-f]{64}", proxies.copy_device_id("ab" * 32, 1, pattern=r"[0-9a-f]{64}")) + + def test_the_copy_id_keeps_the_original_shape(self): + assert len(proxies.copy_device_id("abc", 3)) == 3 + assert re.fullmatch(r"[0-9A-F]{6}", proxies.copy_device_id("ABCDEF", 3)) + assert re.fullmatch(r"[0-9a-f]{6}", proxies.copy_device_id("abcdef", 3)) + + def test_every_catalog_device_id_is_marked_per_device(self): + from app import catalog + + catalog.load_services() + for slug, key in [("proxyrack", "UUID"), ("earnapp", "EARNAPP_UUID"), ("spide", "SPIDE_MACHINE_ID")]: + env = (catalog.get_service(slug) or {}).get("docker", {}).get("env", []) + assert any(v.get("key") == key and v.get("per_device") is True for v in env), slug + # --------------------------------------------------------------------------- # Traffic cap @@ -781,6 +837,43 @@ async def go(): } ] + def test_a_copy_of_a_linked_device_runs_and_registers_its_own_id(self, db): + original = "cd" * 32 + svc = { + "slug": "proxyrack", + "name": "ProxyRack", + "status": "active", + "docker": { + "image": "proxyrack/pop", + "env": [{"key": "UUID", "per_device": True}, {"key": "API_KEY", "secret": True}], + }, + "device_link": {"id_env": "UUID", "steps": "Add the device."}, + } + recorded = {"image": "proxyrack/pop", "env": {"UUID": original, "API_KEY": "k"}} + call = AsyncMock(return_value={"status": "deployed", "exit": {"ip": "93.184.215.14", "country": "US"}}) + register = AsyncMock(return_value=True) + + async def go(): + created = await main.api_create_proxy(_Req(_socks()), {}) + with ( + patch.object(main.catalog, "get_service", return_value=svc), + patch.object(main, "_resolve_worker_id", new_callable=AsyncMock, return_value=7), + patch.object(main, "_worker_features", new_callable=AsyncMock, return_value={"proxy_tunnel"}), + patch.object(main.database, "get_deployment_spec", new_callable=AsyncMock, return_value=recorded), + patch.object(main.database, "list_workers", new_callable=AsyncMock, return_value=[]), + patch.object(main, "_proxy_to_worker", call), + patch.object(main, "_register_device", register), + ): + return await main.api_add_proxied_copy( + created["id"], main.ProxyCopyRequest(worker_id=7, slug="proxyrack"), {} + ) + + result = _run(go) + sent_id = call.call_args.kwargs["json"]["earner"]["env"]["UUID"] + assert sent_id != original and len(sent_id) == 64 + assert register.call_args.args[:2] == ("proxyrack", sent_id) + assert result["device_link"]["id"] == sent_id and result["device_link"]["registered"] is True + def test_dropping_the_cap_lifts_the_stop_and_starts_the_copies(self, db): call = AsyncMock(return_value={})