diff --git a/.changeset/backup-destination-check.md b/.changeset/backup-destination-check.md new file mode 100644 index 0000000..458bbc3 --- /dev/null +++ b/.changeset/backup-destination-check.md @@ -0,0 +1,5 @@ +--- +'@wpelevator/tug': minor +--- + +Add `tug backup destination check`, which opens the control repository and the repository of every project with a backup target from the control host, creates missing ones and exits nonzero when any fails. `tug backup destination set` runs the check after saving (skip it with `--no-check`). Backup destinations may now use any built-in restic backend: `s3:`, `rest:`, `b2:`, `azure:`, `gs:` and `swift:`. `tug backup destination set` keeps exit code 0 when the check fails, since the destination is saved. Adds `POST /api/v1/backup/destination/check` and `deployed` in the repository password response. diff --git a/.changeset/rest-repository-probe.md b/.changeset/rest-repository-probe.md new file mode 100644 index 0000000..590f433 --- /dev/null +++ b/.changeset/rest-repository-probe.md @@ -0,0 +1,5 @@ +--- +'@wpelevator/tug': patch +--- + +Decide whether a backup repository is missing from restic's exit code 10 instead of its message, in control and on nodes. Control's own backup now creates the `_control` repository on a REST server, and a mistyped S3 bucket fails instead of being created. The protection against creating a mistyped bucket applies once each node's agent is upgraded, because nodes run the check themselves. Automatic repository creation now requires restic 0.17 or newer, which matters only with a custom `TUG_RESTIC_IMAGE`; the error names a restic older than 0.17. `backup.check` no longer initializes a repository, and a wrong repository password is named in the error. `sftp:` and `rclone:` destinations were accepted but never worked, so rejecting them with the reason is a fix. `tug backup password` and the repository password endpoint work before a project is deployed and say whether it has been, and `tug backup run` explains that a project that was never deployed has no backup targets yet. diff --git a/README.md b/README.md index ed1409f..e6212fb 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Managed ingress balances HTTP requests across serving replicas with round robin. | --- | --- | --- | | Get images to servers | Builds your images on control (or on your machine with `tug deploy --build-local`) and sends them over connections the agents already make. Or push to any HTTPS registry and declare `image:`, and control pins the tag to a digest before deploying. A failed build leaves the nodes untouched. | [Control plane](docs/control-plane.md#image-delivery-without-a-registry) | | Serve HTTPS | Managed Caddy serves a hostname with `tls: auto`, gets the certificate and renews it. DNS must point at the server and ports 80 and 443 must be reachable. | [Ingress](docs/ingress.md) | -| Back up and restore | Scheduled [restic](https://restic.net) backups of volumes and databases to S3-compatible storage. `tug restore run db` brings one back. Control backs itself up every night. | [Backups](docs/backups.md) | +| Back up and restore | Scheduled [restic](https://restic.net) backups of volumes and databases to S3-compatible storage, a restic REST server and other restic backends. `tug restore run db` brings one back. Control backs itself up every night. | [Backups](docs/backups.md) | | Keep secrets out of git | `tug secret set` keeps them encrypted on control and delivers each only to the nodes whose services use it. Every `${NAME}` in your Compose file comes from control, never from the shell or `.env` of whoever runs the deploy. | [Where values come from](docs/compose.md#where-values-come-from) | | Roll back | `tug rollback` returns to a previous deployment, and a top-level `x-tug: { rollback: auto }` does it when a node reports the new version unhealthy. It cannot undo database writes or schema changes. | [Health checks](docs/healthchecks.md) | | Update without a restart | `x-tug: { rollout: { strategy: rolling } }` starts the new revision beside the old one and switches ingress once it is healthy. It needs ingress, a tug health check and one published port. The default `recreate` restarts the container and can interrupt traffic. | [Rolling rollouts](docs/compose.md#rolling-rollouts) | diff --git a/docs/access-control.md b/docs/access-control.md index 8b61660..4052f47 100644 --- a/docs/access-control.md +++ b/docs/access-control.md @@ -167,6 +167,7 @@ The project column says how a scoped credential's allowed projects are checked: | `GET /api/v1/backup/destination` | `admin` | Control-wide; URL and variable names, never values | | `POST /api/v1/backup/destination` | `admin` | Control-wide | | `DELETE /api/v1/backup/destination` | `admin` | Control-wide | +| `POST /api/v1/backup/destination/check` | `admin` | Control-wide; opens the destination's repositories and creates missing ones | | `GET /api/v1/backup/repository-password` | `admin` | Query `project` prints that project's repository password; `control` prints the control repository password | | `POST /api/v1/secrets` | `admin` | Control-wide | | `DELETE /api/v1/secrets/:name` | `admin` | Control-wide | diff --git a/docs/backups.md b/docs/backups.md index ce13243..61103bf 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -1,15 +1,16 @@ # Backups and recovery -Mark a service or a volume with `x-tug.backup`, give control one S3-compatible destination, and agents run [restic](https://restic.net) on the node that holds the data on a schedule. There is no backup container to build or keep alive. +Mark a service or a volume with `x-tug.backup`, give control one restic repository location (S3-compatible storage, a REST server and other [destinations](#destinations)), and agents run [restic](https://restic.net) on the node that holds the data on a schedule. There is no backup container to build or keep alive. ## Set up your first backup -You need S3-compatible storage reachable from the workload node, and credentials allowed to use the bucket. Create the bucket first. Mark each target in your Compose file as shown in [Compose extensions](compose.md#backups), and see the [WordPress example](https://github.com/wpelevator/tug/blob/main/examples/wordpress/README.md) for a database and files together. A top-level schedule only supplies defaults and enables nothing. +You need storage restic can use, reachable from the control host and from the workload node, and credentials allowed to use it. The example uses S3-compatible storage, see [Destinations](#destinations) for the others. Create the bucket first. Mark each target in your Compose file as shown in [Compose extensions](compose.md#backups), and see the [WordPress example](https://github.com/wpelevator/tug/blob/main/examples/wordpress/README.md) for a database and files together. A top-level schedule only supplies defaults and enables nothing. ```sh # Supply the values through your shell or secret manager. Do not commit them. export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY tug backup destination set s3:https://s3.example.com/bucket/tug-backups --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY +tug backup destination check tug deploy tug backup run tug backup list --check @@ -18,6 +19,8 @@ tug backup password | Command | What it tells you | | --- | --- | +| `backup destination set` | Saves the destination, then checks it from the control host. A failed check is a warning and the exit code stays 0 (`--no-check` skips the check) | +| `backup destination check` | Opens the control repository and each project's repository, creates missing ones and exits nonzero if any fails. Skips project repositories when `_control` fails | | `backup run` | Waits for the configured targets and reports failure | | `backup list --check` | Whether each target succeeded, and exits nonzero if not | | `backup password` | The repository and its password. Store it, with the storage credentials, off the server | @@ -27,6 +30,51 @@ The password is derived from control's master key, so losing both the password a Control also backs itself up to the same destination every night, see [Control plane backup](#control-plane-backup). Configure [signed webhooks](observability.md#alerting) for backup failures, missed protection and recovery notifications. Delivery is best effort; keep an independent scheduled `tug backup list --check`. +## Destinations + +A destination is a restic repository location. `tug backup destination set ` accepts these forms, and each project and control get their own repository below it. Credentials are never part of the URL: pass them as `--env` variables, which control stores encrypted. `RESTIC_PASSWORD`, `RESTIC_REPOSITORY` and `RESTIC_PASSWORD_FILE` are managed by tug and refused. + +| URL form | Credentials (`--env`) | Status | +| --- | --- | --- | +| `s3:https://host/bucket[/prefix]` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` (required) | Exercised on real hosts | +| `rest:http://host:8000/path` | `RESTIC_REST_USERNAME`, `RESTIC_REST_PASSWORD`, or none for a server started with `--no-auth` | Verified against restic's `rest-server` | +| `b2:bucket:path` | `B2_ACCOUNT_ID`, `B2_ACCOUNT_KEY` | Accepted, not yet exercised in a real-host run | +| `azure:container:/path` | `AZURE_ACCOUNT_NAME` with `AZURE_ACCOUNT_KEY` or `AZURE_ACCOUNT_SAS` | Accepted, not yet exercised in a real-host run | +| `gs:bucket:/path` | `GOOGLE_PROJECT_ID` with `GOOGLE_ACCESS_TOKEN`, or `GOOGLE_APPLICATION_CREDENTIALS` | Accepted, not yet exercised in a real-host run. A credentials file path must exist inside the helper container, which tug cannot provide, so prefer `GOOGLE_ACCESS_TOKEN` | +| `swift:container:/path` | The `OS_*` variables restic documents for your Keystone setup | Accepted, not yet exercised in a real-host run | +| `/absolute/path` | None | Development only: the path is read on each node and is not shared between nodes | + +Each backend's variables are listed in restic's [Preparing a new repository](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html). + +### REST server + +A [rest-server](https://github.com/restic/rest-server) is the simplest destination you host yourself: + +```sh +export RESTIC_REST_USERNAME RESTIC_REST_PASSWORD +tug backup destination set rest:http://backup.internal:8000/tug --env RESTIC_REST_USERNAME --env RESTIC_REST_PASSWORD +``` + +Plain HTTP is acceptable only on a private network or VPN that the control host and every node share. Use `rest:https://` anywhere else. A server started with `--no-auth` needs no `--env`. + +### Not supported + +`sftp:` is rejected because tug cannot give the restic helper container SSH keys or a `known_hosts` file. Run a rest-server and reach it over a private network or VPN, or use S3. `rclone:` is rejected because the helper image does not configure rclone. + +### Check the destination + +```sh +tug backup destination check +``` + +The check runs from the control host in a restic helper container. It opens `_control` and the repository of every project with an enabled backup target. An existing repository is `ok`. A missing one, which restic reports with exit code 10, is created with `restic init` and reported as `initialized`, which also proves the credentials can write. Anything else is `error` with restic's message: a wrong repository password (exit code 12), a bucket that does not exist, a refused login or an unreachable store. A bucket is never created by a check or a backup, so a typo in the bucket name fails instead. When `_control` fails, the project repositories are `skipped`, because they share the destination and would fail the same way, each after its own timeout. The command exits nonzero when any repository fails, and `tug backup destination show` returns the last result as `checkResult`. A check refuses to start while control's own backup runs or another check is running, and the whole check stops after four minutes. + +`initialized` means the check created a new, empty repository. During a [migration](migration.md#coming-from-an-existing-restic-setup) an unexpected `created a new, empty repository` means the destination path is wrong, because the repository you expected would have been found. + +`tug backup destination set` runs the same check right after saving. The destination stays saved and the exit code stays 0 when the check fails: stderr says `saved; the check failed` and names the failing repositories, and you fix the cause and run `tug backup destination check`. A control too old to check destinations prints a warning instead. Pass `--no-check` to skip it. Nodes connect to the destination at their first backup, so a passing check proves the control host's access, not that every node can reach the destination. + +Repositories are created by the first check or backup only when restic is 0.17 or newer, the first version that reports a missing repository with exit code 10. A failure from an image older than 0.17 says so. The stock image is new enough, so this matters only with a custom `TUG_RESTIC_IMAGE`. + ## Plan for recovery Workload snapshots hold application data. Control archives hold tug state and keys. Store both away from the VPS, and keep their credentials and passwords somewhere you can reach if the server is lost. @@ -107,6 +155,8 @@ backup: | `backup.no_destination` | No destination is configured | | `backup.cancelled` | Someone ran `tug backup cancel `. A cancelled backup is a warning and is not retried | +Exit status is the signal to wire into monitoring: `tug backup list --check` exits nonzero when a target is stale, failed or missed, or when nothing is protected. Run it from cron or a monitor, and keep running it during a [migration](migration.md#coming-from-an-existing-restic-setup), when the old backup system may be failing without anyone noticing. + While control is down no backup starts, which is the price of one scheduler. Running jobs finish and report when control returns. `tug backup run` takes a backup now, and `tug backup list` shows the catalog, see [CLI](cli.md#backups). ## Restore @@ -179,7 +229,7 @@ As soon as a backup destination exists, control backs itself up with nothing mor | Who runs it | Control runs the restic helper on its own Docker socket, so the control host needs no node and no cron job | | Stalls | restic retries a failing store (a missing bucket, a refused login, an unreachable host) for many minutes. When it prints only retry notices for two minutes (`TUG_CONTROL_BACKUP_STALL_MS`), the backup fails with restic's final message and last cause, which `tug backup list` shows | -Create the bucket first. A repository is initialized only when restic's final answer is that it does not exist, so a bucket created while a backup was retrying is picked up by the next attempt. +Create the bucket first. A repository is initialized only when restic exits with code 10 (the repository does not exist). A missing bucket makes restic retry and fail, so a bucket created while a backup was retrying is picked up by the next attempt. `tug backup list` shows a `control` line with the last success in the destination now in use. Switching destination starts a first backup there and reports it as unprotected until that backup completes. A local-path destination (for development) is labelled `local path only: not off-host`. `tug backup list --check` fails when the last control backup failed, when none succeeded for twice the interval (two days by default), or when a destination has had no completed backup an hour after it was set. `tug events --project _control` shows `backup.completed` and `backup.failed`. @@ -224,7 +274,8 @@ A node runs backup and restore jobs in lanes: a restore and its safety backup fi | Control down | No backup starts. Running jobs finish and report when control returns, and a long gap yields one missed event and one catch-up run per target | | Consistency | Live volumes are crash-consistent, not application-consistent. Use a command backup for anything with its own consistency | | One node per volume | A volume is backed up from the node that holds it. A volume mounted from a file server with `driver_opts` is read through that node's mount, so back up the file server's own disk where you can | -| Not built | Repository maintenance (`restic check`, a read-verify of the data) and SFTP destinations | +| Not built | Repository maintenance (`restic check`, a read-verify of the data) | +| Rejected destinations | `sftp:` and `rclone:` URLs, see [Not supported](#not-supported) | | Leftovers | A scratch restore leaves its volume (`tug_r__`) behind, and the safety snapshot of every in-place restore is kept for ever. No command lists or removes either. Remove scratch volumes with `docker volume rm` on the node, and safety snapshots with `restic forget` using the repository from `tug backup password` | | Helper memory | The restic helper defaults to 512 MB, half of a 1 GB host. Lower it with `memory` under `x-tug.backup`. Larger repositories need more, so test the limit against your data | | `tug exec` | Cancelling it does not stop the remote process | @@ -244,6 +295,6 @@ Backup destination settings: the control plane uses one destination, and `tug ba | Variable | Where | Default | Purpose | | --- | --- | --- | --- | | `TUG_BACKUP_TICK_MS` | Control | `30000` | Scheduler tick | -| `TUG_RESTIC_IMAGE` | Control | `restic/restic:0.19.1` | restic helper image | +| `TUG_RESTIC_IMAGE` | Control | `restic/restic:0.19.1` | restic helper image. Must be 0.17 or newer, the first release that exits with code 10 for a missing repository; older images cannot create repositories automatically | | `TUG_BACKUP_CONCURRENCY` | Agent | `2` on 3 GiB or more, else `1` | Concurrent backup jobs per node | | `TUG_CONTROL_BACKUP_INTERVAL_MS`, `TUG_CONTROL_BACKUP_RETRY_MS`, `TUG_CONTROL_BACKUP_STALL_MS`, `TUG_CONTROL_BACKUP_LIMIT_MS`, `TUG_CONTROL_BACKUP_DIR` | Control | See [Configuration](configuration.md#control-plane) | Control backup timing and location | diff --git a/docs/cli.md b/docs/cli.md index 86ea9d6..e9431f3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -381,13 +381,14 @@ With no node, both commands offer the enabled ingress nodes in a terminal and ne | Command | Options | What it does | | --- | --- | --- | -| `tug backup destination set ` | `--env NAME` (repeatable) | Sets where backups go, for example `s3:https://s3.example.com/bucket/tug-backups` with `--env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY`. Each value is read from the environment variable of that name, else from a stdin line, never an argument. S3 needs both AWS variables. Values are stored encrypted and released only to a node running a backup. An absolute path works for development but is read per node and not shared. Replaces the previous destination | -| `tug backup destination show` | | The URL and variable names, never values | +| `tug backup destination set ` | `--env NAME` (repeatable) | Sets where backups go, for example `s3:https://s3.example.com/bucket/tug-backups` with `--env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY`. Accepts `s3:`, `rest:`, `b2:`, `azure:`, `gs:`, `swift:` and an absolute path, and rejects `sftp:` and `rclone:`, see [Destinations](backups.md#destinations). Each value is read from the environment variable of that name, else from a stdin line, never an argument. S3 needs both AWS variables. Values are stored encrypted and released only to a node running a backup. An absolute path works for development but is read per node and not shared. Replaces the previous destination. Afterwards it checks the destination from the control host like `destination check` and prints a failed check as a warning on stderr, exiting 0 because the destination is saved. `--no-check` skips the check | +| `tug backup destination show` | | The URL and variable names, never values, and the last check as `checkedAt` and `checkResult` | +| `tug backup destination check` | | Runs from the control host: opens the control repository and each project's repository, creates the ones that do not exist and prints a result per repository. Skips project repositories when `_control` fails. Exits nonzero when any fails. It does not prove that every node can reach the destination. Needs `admin` | | `tug backup destination remove` | | Removes the destination and its credentials. Existing backups stay in the repository | | `tug backup run [targets...]` | `--no-wait`, `--timeout ` (default `3600`), `--select` | Backs up the named services or volumes now, or every target, and waits. Exits nonzero if any backup fails or is missed. Needs `deploy` | | `tug backup list` | `--limit ` (recent runs, default `20`), `--check`, `--allow-empty` | Shows each target with node, schedule, last success and next run, then recent runs. Needs `read` | | `tug backup cancel ` | | Stops a queued or running backup. The run id is the last column of `backup list`. It is reported as cancelled and not retried | -| `tug backup password` | `--control` | Prints the project's repository and password, or with `--control` the control repository and password, the one value to keep off the server for recovering control | +| `tug backup password` | `--control` | Prints the project's repository and password (also before the project is deployed, with a warning, since the password derives from the name), or with `--control` the control repository and password, the one value to keep off the server for recovering control | | `tug restore run [target]` | `--from `, `--scratch`, `--no-safety-backup`, `--writers `, `--no-writers`, `--confirm`, `--timeout ` (default `3600`), `--select-snapshot`, `--select-writers` | Restores a service or volume. Without `--confirm` it prints the plan and changes nothing. `--from` takes a backup run id, a restic snapshot id or `latest`. It waits up to five minutes for stopped services to serve again, then finishes with a warning. See [Restore](backups.md#restore) | | `tug restore list` | | Recent restores | | `tug restore cancel ` | | Stops a restore that is waiting on its safety backup (nothing has changed) or restoring (it is interrupted and its services stay stopped until `resume` or `release`) | diff --git a/docs/migration.md b/docs/migration.md index 4f537b8..8061391 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -49,6 +49,35 @@ tug node add root@NEW --name web-2 --address 10.0.0.9 --label web --install-dock Nodes must carry all of a service's labels, and nodes whose agent has not been seen recently are skipped when labels are resolved. Keep explicit `nodes:` only where a service must stay on one named host, such as a database that owns a local volume. See [Compose extensions](compose.md#service-x-tug-keys) and [Nodes](nodes.md#node-labels). `tug scale` stays within a service's explicit `placement.nodes`, so change the placement and deploy to move a pinned service. +## Coming from an existing restic setup + +tug derives each repository password from control's master key and manages `RESTIC_PASSWORD` and `RESTIC_REPOSITORY` itself, so it cannot open a repository that has a password of your choosing. Point tug at a new path beside the old repository, for example `rest:http://backup.internal:8000/tug` next to `rest:http://backup.internal:8000/old`, and let it start fresh: + +```sh +tug backup destination set rest:http://backup.internal:8000/tug --env RESTIC_REST_USERNAME --env RESTIC_REST_PASSWORD +tug backup password +``` + +To bring old snapshots along, copy them in with the passwords of both repositories (`tug backup password` prints the new one). The target repository must exist first. Deploy the project and run `tug backup destination check`, which creates it, or create it yourself with Tug's password so deduplication carries over, before Tug creates it: + +```sh +restic -r rest:http://backup.internal:8000/tug/ init --copy-chunker-params --from-repo rest:http://backup.internal:8000/old +``` + +Then copy: + +```sh +restic -r rest:http://backup.internal:8000/tug/ copy --from-repo rest:http://backup.internal:8000/old +``` + +To adopt an existing repository instead, place it at `/` and add tug's password as an extra key, which leaves the old password working: + +```sh +restic -r key add --new-password-file +``` + +Copied and adopted snapshots carry other hosts and tags than tug's, so `tug backup list`, retention and `tug restore` ignore them. Restore those with plain restic. Keep running `tug backup list --check` while both systems exist, because the old one may be failing without anyone noticing. See [Destinations](backups.md#destinations). + ## Move the web tier A web tier whose database and uploads live elsewhere is stateless, so it is the cheapest move. diff --git a/docs/roadmap.md b/docs/roadmap.md index 74834db..7ed1d7d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -114,7 +114,7 @@ The stateful half of principle 5, and something a central control plane does bet - **Done: generic backups.** `x-tug.backup` marks a service or a volume. tug has no database support of its own: a service backup is a `command` run in the running container whose output is stored (database recipes are documentation), a volume backup archives its files. Control schedules them as agent commands on the placement node; the agent streams into restic in a helper container, uploading to a destination configured on control (S3-compatible, encrypted before leaving the node), one repository per project. Retention, a catalog, `backup run` and `backup list`, and `backup.*` events for completed, failed, missed and stale runs. See [Backups](backups.md). - **Done: restore.** `tug restore` locks the project, stops the services that write to the target, takes a safety backup, restores through the service's own `restore` command or the volume's files, and starts them again; an interrupted restore keeps the writers stopped until resumed or released. A scratch restore of a volume touches nothing live. - **Done: `tug run`.** `tug run SERVICE -- cmd` runs a one-off container with the service's image, environment, secrets, network and mounts, returns the exit code and is audited. -- Not done: `x-tug.release` (a command that runs once per deployment before or after rollout, recorded on the deployment, a failure stopping the rollout), SFTP and postgres-free extras such as a weekly `restic check`/prune, a restore drill, `destination test`, and failed or missed backups as webhooks (M8). +- Not done: `x-tug.release` (a command that runs once per deployment before or after rollout, recorded on the deployment, a failure stopping the rollout), postgres-free extras such as a weekly `restic check`/prune and a restore drill, and failed or missed backups as webhooks (M8). - Moving a stateful service to another node (`tug move SERVICE --to NODE`) as backup, stop, restore and re-place, since the pieces are the same. ### M12 — Container networking over the mesh (cancelled) @@ -186,7 +186,7 @@ Claims must match verified behavior. Image rollback does not undo data, schema o 4. **Now, ordered by operator value** (working file `plan-operator-value.md`; a hands-on VM run and an outside product review on 9 November 2026 independently reached the same order): 1. **Move bulk data in and out.** End-to-end backpressure for `exec`/`run` stdin and stdout (every live-channel hop still caps at 1 MiB, and a slow reader cuts a download far sooner), tested at 100 MB in both directions with a hash check; a documented way to move a database or a directory, and a "move an existing site in" guide. 2. **Hear first, and say what is true.** Actionable notifications for failed, missed or stale backups, crash loops, failed or rolled-back deployments and stale nodes; an optional external HTTP probe per ingress host distinguishes service health from user reachability. Keep notifications actionable and include the affected resource, observed failure, and next operator check. - 3. **Backups you can trust.** `backup list --check` fails when nothing is protected, a protection level per target (local, off-host, verified, restore-tested), `restic check` and a destination test, cleanup of scratch volumes and safety snapshots, and fault exercises for destination outage, damaged repository, crash during control activation and agents reconnecting after an older snapshot. The historical QEMU recovery drill covers lost workload and control hosts; it is evidence for those paths, not these additional failure scenarios. + 3. **Backups you can trust.** `backup list --check` fails when nothing is protected, a protection level per target (local, off-host, verified, restore-tested), `restic check` (the destination check, `tug backup destination check`, is done), cleanup of scratch volumes and safety snapshots, and fault exercises for destination outage, damaged repository, crash during control activation and agents reconnecting after an older snapshot. The historical QEMU recovery drill covers lost workload and control hosts; it is evidence for those paths, not these additional failure scenarios. 4. **Staging that is more than a pattern.** `tug project delete` and a cross-project data copy. 5. **Show the operator's scope.** Label control-wide views, show the resolved control and project before high-impact commands, give `ps`, `events` and `doctor` a compact human default. 6. **Done: control is the only source of deployment values.** Operators could not tell whether `${NAME}` came from control, their shell or `.env`. Now every value comes from the project store (secrets through `tug secret`, visible values such as image tags, hosts and ports through `tug var`, one name space), an explicit `deploy --var`, or the default in the file; the shell, `.env`, `env_file` and `COMPOSE_PROFILES` are ignored, `deploy --dry-run` and the deployment record show each value's source, and `secret import` / `var import` move an existing `.env` in ([Where values come from](compose.md#where-values-come-from), [upgrade note](releases.md#upgrading-to-30-values-come-only-from-control)). A `major`. @@ -245,7 +245,8 @@ The test layers, the metrics milestones are held to and the backlog of mocked te - [x] M11 `x-tug.backup` (generic: a command in a service, or a volume's files), scheduled backups to S3-compatible storage with restic, catalog, `backup run`, `backup list`, `restore` (verified against real Docker and on two real hosts with an S3-compatible server and a scheduled run: [Real-host evaluation](real-host-evaluation.md#generic-backups-restore-and-tug-run-on-fresh-hosts-1-november-2026); not yet against AWS S3) - [x] M11 `tug run` one-off containers - [ ] M11 `x-tug.release` run-once hooks -- [ ] M11 weekly maintenance (`restic check`, prune), restore drill, `backup-destination-test`, SFTP +- [ ] M11 weekly maintenance (`restic check`, prune), restore drill +- [x] M11 `backup destination check` (and `destination set` checks by default); SFTP and rclone destinations are rejected for now because the helper has no SSH keys or rclone configuration - [x] M11 `restore` waits for the restored writers to be serving again (five minutes, then it finishes with a warning) - [ ] M11 list and remove scratch-restore volumes, and expire safety snapshots (they accumulate for ever and only `restic forget` removes them) - [x] M11 `backup list --check`: exit nonzero when any target is stale, failed or missed, as an interim for cron and monitors until webhooks (M8) exist diff --git a/src/agent/jobs.test.ts b/src/agent/jobs.test.ts index 0af58b4..bc49278 100644 --- a/src/agent/jobs.test.ts +++ b/src/agent/jobs.test.ts @@ -71,41 +71,75 @@ test('a check job reports a healthy repository and removes its helper container' assert.deepEqual(docker.names(), [], 'the helper container is removed after the job'); }); -test('a missing repository is initialized, but a wrong password is not', async () => { - const missing = dockerWith((cmd) => { - if (cmd.join(' ') === 'restic cat config') - return { - code: 10, - err: 'Fatal: unable to open config file\nIs there a repository at the following location?', - }; - return { code: 0 }; - }); - const first = await runner(missing); - await first.jobs.start(command('cmd_1', payload())); - await settle(first.jobs, 1); - assert.equal( - (first.jobs.pending()[0].result as { repository?: string }).repository, - 'initialized', - 'a missing repository is created', +const REST_MISSING = + 'Fatal: repository does not exist: unable to open config file: does not exist\nIs there a repository at the following location?\nrest:http://backup.internal:8030/tug/site/'; +const S3_NO_KEY = + 'Fatal: repository does not exist: unable to open config file: Stat: The specified key does not exist.\nIs there a repository at the following location?\ns3:http://s3.example.com/exists/tug'; +const BUCKET_MISSING = + 'Fatal: unable to open config file: Stat: The specified bucket does not exist.\nIs there a repository at the following location?'; + +async function finished(docker: FakeDocker, kind: string) { + const { jobs } = await runner(docker); + await jobs.start(command('cmd_1', payload(), kind)); + await settle(jobs, 1); + return jobs.pending()[0]; +} +const ran = (docker: FakeDocker, cmd: string) => + docker.execOptions.some((e) => e.cmd.join(' ') === cmd); + +test('a run job initializes a repository restic reports as missing with exit 10, on REST and S3', async () => { + for (const [what, err] of [ + ['a REST server', REST_MISSING], + ['an S3 key that does not exist', S3_NO_KEY], + ] as const) { + const docker = dockerWith((cmd) => + cmd.join(' ') === 'restic cat config' ? { code: 10, err } : { code: 0 }, + ); + const result = await finished(docker, 'backup.run'); + assert.ok( + !/repository check failed|repository init failed/.test(JSON.stringify(result.result)), + `${what} without a repository must not fail the repository check`, + ); + assert.ok(ran(docker, 'restic init'), `${what}: restic init ran`); + } +}); + +test('a check job only reports a missing repository and never initializes it', async () => { + const docker = dockerWith((cmd) => + cmd.join(' ') === 'restic cat config' ? { code: 10, err: REST_MISSING } : { code: 0 }, ); - assert.ok( - missing.execOptions.some((e) => e.cmd.join(' ') === 'restic init'), - 'restic init ran', + const result = await finished(docker, 'backup.check'); + assert.equal(result.state, 'succeeded', 'a missing repository is not a failed check'); + assert.equal( + (result.result as { repository?: string }).repository, + 'missing', + 'the check reports the repository as missing', ); + assert.ok(!ran(docker, 'restic init'), 'backup.check is read-only'); +}); - const wrong = dockerWith((cmd) => - cmd.join(' ') === 'restic cat config' - ? { code: 1, err: 'Fatal: wrong password or no key found' } - : { code: 0 }, - ); - const second = await runner(wrong); - await second.jobs.start(command('cmd_2', payload())); - await settle(second.jobs, 1); - assert.equal(second.jobs.pending()[0].state, 'failed', 'a wrong password fails the job'); - assert.ok( - !wrong.execOptions.some((e) => e.cmd.join(' ') === 'restic init'), - 'a wrong password must never initialize a new repository', +test('a missing bucket (exit 1) and a wrong password (exit 12) fail a run job without init', async () => { + for (const [what, code, err, expected] of [ + ['a missing bucket', 1, BUCKET_MISSING, /bucket does not exist/], + ['a wrong password', 12, 'Fatal: wrong password or no key found', /wrong repository password/], + ] as const) { + const docker = dockerWith((cmd) => + cmd.join(' ') === 'restic cat config' ? { code, err } : { code: 0 }, + ); + const result = await finished(docker, 'backup.run'); + assert.equal(result.state, 'failed', `${what} fails the job`); + assert.match(JSON.stringify(result.result), expected, `${what} is explained`); + assert.ok(!ran(docker, 'restic init'), `${what} must never create a repository or bucket`); + } +}); + +test('a restore job fails on a missing repository instead of creating one', async () => { + const docker = dockerWith((cmd) => + cmd.join(' ') === 'restic cat config' ? { code: 10, err: REST_MISSING } : { code: 0 }, ); + const result = await finished(docker, 'backup.restore'); + assert.equal(result.state, 'failed', 'there is nothing to restore from'); + assert.ok(!ran(docker, 'restic init'), 'a restore never initializes'); }); test('error text never contains credentials', async () => { diff --git a/src/backup/control-backup.test.ts b/src/backup/control-backup.test.ts index 9c35e5c..ca9afaf 100644 --- a/src/backup/control-backup.test.ts +++ b/src/backup/control-backup.test.ts @@ -6,8 +6,8 @@ import path from 'node:path'; import { controlRepository, controlRepositoryPassword, + checkRepository, fetchControlArchive, - isMissingRepository, listControlSnapshots, pushControlArchive, } from './control-backup.ts'; @@ -69,7 +69,7 @@ test('a push creates a missing repository, streams the archive into restic and r await fs.writeFile(archive, 'archive-bytes'); const docker = dockerWith((cmd) => { const line = cmd.join(' '); - if (line === 'restic cat config') return { code: 1, err: NEVER_CREATED }; + if (line === 'restic cat config') return { code: 10, err: NEVER_CREATED }; if (line.startsWith('restic backup')) return { code: 0, out: '{"message_type":"summary","snapshot_id":"abc123def456"}\n' }; return { code: 0 }; @@ -159,7 +159,7 @@ test('a fetch writes the newest control snapshot to a private file and reports a ); const empty = dockerWith((cmd) => - cmd[1] === 'cat' ? { code: 1, err: NEVER_CREATED } : { code: 0 }, + cmd[1] === 'cat' ? { code: 10, err: NEVER_CREATED } : { code: 0 }, ); await assert.rejects( () => fetchControlArchive(empty, S3, path.join(dir, 'none.tar.gz')), @@ -237,29 +237,13 @@ test('a hung restic fails at the time limit and its helper is removed', async () const BUCKET_MISSING = 'Fatal: unable to open config file: Stat: The specified bucket does not exist.\nIs there a repository at the following location?'; +/** restic 0.19.1 against an S3 bucket that exists but holds no repository: exit code 10. */ const NEVER_CREATED = - 'Fatal: unable to open config file: Stat: The specified key does not exist.\nIs there a repository at the following location?'; - -test('only a repository that was never created counts as missing', () => { - assert.equal( - isMissingRepository(NEVER_CREATED), - true, - 'a missing config file is a repository to create', - ); - for (const [what, text] of [ - ['a wrong bucket', BUCKET_MISSING], - [ - 'a refused login', - 'Stat: AccessDenied: Access Denied.\nIs there a repository at the following location?', - ], - [ - 'an unreachable store', - 'dial tcp: lookup s3.example.com: no such host\nIs there a repository at the following location?', - ], - ['a wrong password', 'wrong password or no key found'], - ] as const) - assert.equal(isMissingRepository(text), false, `${what} is not a missing repository`); -}); + 'Fatal: repository does not exist: unable to open config file: Stat: The specified key does not exist.\nIs there a repository at the following location?\ns3:http://s3.example.com/bucket/tug'; +/** restic 0.19.1 against a rest-server without the repository: exit code 10. */ +const REST_MISSING = + 'Fatal: repository does not exist: unable to open config file: does not exist\nIs there a repository at the following location?\nrest:http://backup.internal:8030/tug/_control/'; +const WRONG_PASSWORD = 'Fatal: wrong password or no key found'; test('a wrong bucket fails a push and a fetch instead of creating or reporting an empty repository', async () => { const dir = await scratch(); @@ -423,7 +407,7 @@ test('listing reads the snapshots from the destination, newest first, flat or gr test('listing reports an empty destination as missing and an unreadable one as an error', async () => { const empty = dockerWith((cmd) => - cmd[1] === 'cat' ? { code: 1, err: NEVER_CREATED } : { code: 0 }, + cmd[1] === 'cat' ? { code: 10, err: NEVER_CREATED } : { code: 0 }, ); assert.deepEqual( await listControlSnapshots(empty, S3), @@ -496,32 +480,8 @@ test('a named snapshot is restored only when control stored it, and the id is va } }); -const REST_404 = - 'Fatal: unable to open config file: Stat: unexpected HTTP response (404): 404 page not found\nIs there a repository at the following location?'; const REST_401 = 'Fatal: unable to open config file: Stat: unexpected HTTP response (401): 401 Unauthorized\nIs there a repository at the following location?'; -const LOCAL_MISSING = - 'Fatal: unable to open config file: stat /srv/backups/_control/config: no such file or directory\nIs there a repository at the following location?'; -const HINT_ONLY = - 'Fatal: something nobody has seen\nIs there a repository at the following location?'; - -test('a repository is missing only on positive evidence that its config is absent', () => { - for (const [what, text] of [ - ['an S3 key that does not exist', NEVER_CREATED], - ['a local path that does not exist', LOCAL_MISSING], - ['a REST server answering 404', REST_404], - ] as const) - assert.equal(isMissingRepository(text), true, `${what} is a repository to create`); - for (const [what, text] of [ - ['a REST server refusing the login', REST_401], - ['an answer with only the hint', HINT_ONLY], - ] as const) - assert.equal( - isMissingRepository(text), - false, - `${what} is a backend error, not an empty repository`, - ); -}); test('a refused REST login fails the push and the fetch instead of initializing or reporting nothing', async () => { const dir = await scratch(); @@ -604,35 +564,12 @@ const BUCKET_APPEARED = [ 's3:http://10.0.2.2:9000/tug-backups/_control', ].join('\n'); -test('only the final failure decides whether a repository is missing, not the retries before it', () => { - assert.equal( - isMissingRepository(BUCKET_APPEARED), - true, - 'a bucket that did not exist during earlier attempts and was created since leaves a repository to create', - ); - assert.equal( - isMissingRepository( - [ - retryNotice(8, 'The specified key does not exist'), - 'Fatal: unable to open config file: Stat: The specified bucket does not exist', - ].join('\n'), - ), - false, - 'a final failure about the bucket is a backend error whatever the retries said', - ); - assert.equal( - isMissingRepository(retryNotice(8, 'The specified key does not exist')), - false, - 'retry notices alone are not evidence of anything', - ); -}); - test('a bucket created while restic was retrying lets the first backup initialize the repository', async () => { const dir = await scratch(); const archive = path.join(dir, 'a.tar.gz'); await fs.writeFile(archive, 'x'); const docker = dockerWith((cmd) => - cmd[1] === 'cat' ? { code: 1, err: BUCKET_APPEARED } : { code: 0 }, + cmd[1] === 'cat' ? { code: 10, err: BUCKET_APPEARED } : { code: 0 }, ); try { const result = await pushControlArchive(docker, S3, archive); @@ -757,3 +694,131 @@ test('a slow store that retries and then makes progress is not failed as stalled await fs.rm(dir, { recursive: true, force: true }); } }); + +test('a push initializes a repository that restic reports as missing with exit code 10, whatever the backend says', async () => { + for (const [what, err] of [ + ['a REST server', REST_MISSING], + ['an S3 key that does not exist', NEVER_CREATED], + ] as const) { + const dir = await scratch(); + const archive = path.join(dir, 'a.tar.gz'); + await fs.writeFile(archive, 'x'); + const docker = dockerWith((cmd) => (cmd[1] === 'cat' ? { code: 10, err } : { code: 0 })); + try { + const result = await pushControlArchive(docker, S3, archive); + assert.equal(result.repository, 'initialized', `${what} without a repository is created`); + assert.ok( + execs(docker).some((line) => line.startsWith('restic init')), + `${what}: restic init runs`, + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + } +}); + +test('a wrong password (exit 12) and a missing bucket (exit 1) fail a push without init and explain the cause', async () => { + for (const [what, code, err, expected] of [ + ['a wrong password', 12, WRONG_PASSWORD, /wrong repository password/], + ['a missing bucket', 1, BUCKET_MISSING, /bucket does not exist/], + ] as const) { + const dir = await scratch(); + const archive = path.join(dir, 'a.tar.gz'); + await fs.writeFile(archive, 'x'); + const docker = dockerWith((cmd) => (cmd[1] === 'cat' ? { code, err } : { code: 0 })); + try { + await assert.rejects( + () => pushControlArchive(docker, S3, archive), + expected, + `${what} fails the backup with its cause`, + ); + assert.ok( + !execs(docker).some((line) => line.startsWith('restic init')), + `${what} never triggers restic init`, + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + } +}); + +test('the old-restic hint appears only for a known restic older than 0.17, never for a 0.19 missing bucket', async () => { + for (const [version, hinted] of [ + ['0.19.1', false], + ['0.16.4', true], + ] as const) { + const dir = await scratch(); + const archive = path.join(dir, 'a.tar.gz'); + await fs.writeFile(archive, 'x'); + const docker = dockerWith((cmd) => + cmd[1] === 'cat' + ? { code: 1, err: `${BUCKET_MISSING}\ns3:http://s3.example.com/nope/tug` } + : cmd[1] === 'version' + ? { code: 0, out: `restic ${version} compiled with go1.22\n` } + : { code: 0 }, + ); + try { + await assert.rejects( + () => pushControlArchive(docker, S3, archive), + (error: Error) => + /bucket does not exist/.test(error.message) && + /TUG_RESTIC_IMAGE must be 0\.17 or newer/.test(error.message) === hinted, + `restic ${version}: the hint is ${hinted ? 'present' : 'absent'}`, + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + } +}); + +test('checkRepository redacts credentials from errors the helper itself throws', async () => { + const docker = dockerWith(() => ({ code: 0 })); + docker.failCreate = () => + new Error(`docker said no for ${S3.env.AWS_SECRET_ACCESS_KEY} and repo-password-1`); + const checked = await checkRepository(docker, { + repository: 's3:https://s3.example.com/bucket/tug/shop', + env: S3.env, + password: 'repo-password-1', + resticImage: S3.resticImage, + }); + assert.equal(checked.state, 'error', 'a Docker failure is an error state'); + assert.ok( + !JSON.stringify(checked).includes('sekret-value-1') && + !JSON.stringify(checked).includes('repo-password-1'), + 'neither the access secret nor the repository password appears in the error', + ); +}); + +test('checkRepository reports ok, initialized and error states and never throws for a backend failure', async () => { + const access = { + repository: 's3:https://s3.example.com/bucket/tug/shop', + env: S3.env, + password: 'repo-password-1', + resticImage: S3.resticImage, + }; + const run = (code: number, err: string, initCode = 0) => + checkRepository( + dockerWith((cmd) => + cmd[1] === 'cat' + ? { code, err } + : cmd[1] === 'init' + ? { code: initCode, err: 'init boom' } + : { code: 0 }, + ), + access, + ); + assert.deepEqual(await run(0, ''), { state: 'ok' }, 'an existing repository is ok'); + assert.deepEqual( + await run(10, REST_MISSING), + { state: 'initialized' }, + 'a missing repository is created', + ); + const bucket = await run(1, BUCKET_MISSING); + assert.equal(bucket.state, 'error', 'a missing bucket is an error'); + assert.match(bucket.error ?? '', /bucket does not exist/, 'the error carries restic message'); + const wrong = await run(12, WRONG_PASSWORD); + assert.match(wrong.error ?? '', /wrong repository password/, 'exit 12 explains the password'); + const noInit = await run(10, REST_MISSING, 1); + assert.equal(noInit.state, 'error', 'a failed init is an error'); + assert.match(noInit.error ?? '', /repository init failed/, 'the init failure is named'); +}); diff --git a/src/backup/control-backup.ts b/src/backup/control-backup.ts index b727b20..46943dc 100644 --- a/src/backup/control-backup.ts +++ b/src/backup/control-backup.ts @@ -4,6 +4,7 @@ import { rm } from 'node:fs/promises'; import { pipeline } from 'node:stream/promises'; import { redactSecrets } from './destination.ts'; import { collect, ensureImage, parseSummary } from './job.ts'; +import { classifyRepositoryProbe, describeProbeFailure } from './repository-probe.ts'; import type { Collected } from './job.ts'; import type { AttachedProcess, DockerApi } from '../docker-api.ts'; @@ -32,23 +33,12 @@ const RETENTION = [ '--keep-monthly', '6', ]; -/** - * restic appends "Is there a repository at the following location?" to every failure to open the config, whatever - * the cause, so that line says nothing. A repository counts as missing only on positive evidence that the config - * object itself is absent: S3 `NoSuchKey`, a local `no such file`, an SFTP missing file or a 404 from a REST server. - * Anything else (a wrong bucket, a refused login, an unreachable store, a wrong password, an answer nobody has seen - * before) is a backend error, so it is neither initialized over nor reported as "no control backup". The denylist - * stays as a second guard for messages that mention both. - */ -const CONFIG_IS_ABSENT = - /specified key does not exist|NoSuchKey|no such file or directory|file does not exist|unexpected HTTP response \(404\)|\b404 (page )?not found/i; -const NOT_A_MISSING_REPOSITORY = - /NoSuchBucket|bucket does not exist|AccessDenied|Forbidden|Unauthorized|InvalidAccessKey|SignatureDoesNotMatch|dial tcp|no such host|timeout|timed out|connection refused|connection reset|certificate|wrong password|\(40[13]\)/i; /** * restic keeps retrying a failing backend and prints one notice per attempt ("Stat() returned error, * retrying after 8s: ..."), then ends with its final `Fatal:` message. The notices are history: a bucket that - * did not exist a minute ago and was created since ends in "repository does not exist", and only that final state - * says what to do. Classification and summaries therefore ignore the notices. + * did not exist a minute ago and was created since ends in "repository does not exist". Summaries therefore show the + * final message and reduce the notices to a count. Whether a repository is missing is never read from this text: it is + * restic's exit code ({@link classifyRepositoryProbe}). */ const RETRY_NOTICE = /returned error, retrying after [^:]*:\s*/i; function splitRetries(text: string): { notices: string[]; rest: string[] } { @@ -58,11 +48,6 @@ function splitRetries(text: string): { notices: string[]; rest: string[] } { rest: lines.filter((line) => !RETRY_NOTICE.test(line)), }; } -export const isMissingRepository = (stderr: string): boolean => { - const final = splitRetries(stderr).rest.join('\n'); - return CONFIG_IS_ABSENT.test(final) && !NOT_A_MISSING_REPOSITORY.test(final); -}; - export interface ControlRepositoryAccess { /** The destination URL as stored, such as `s3:https://host/bucket/prefix`. */ url: string; @@ -115,10 +100,40 @@ function summarize(text: string): string { type Exec = (cmd: string[]) => Promise; +/** + * restic's message for a failed probe, with the hint that fixes it. Only an exit code 1 can mean "restic too old to + * say the repository is missing", so only then is the version read. + */ +async function explainProbe(exec: Exec, probe: Collected): Promise { + const detail = summarize(probe.err || probe.out); + if (probe.code !== 1) return describeProbeFailure(probe, detail); + const version = (await exec(['restic', 'version']).catch(() => undefined))?.out.match( + /restic (\d+\.\d+\.\d+)/, + )?.[1]; + return describeProbeFailure(probe, detail, version); +} + +/** One restic repository to open: the control repository, or a project's when checking a destination. */ +interface ResticAccess { + /** The full restic repository, such as `s3:https://host/bucket/prefix/_control`. */ + repository: string; + env: Record; + password: string; + resticImage: string; + limitMs?: number; + stallMs?: number; + signal?: AbortSignal; +} + +const controlAccess = (access: ControlRepositoryAccess): ResticAccess => ({ + ...access, + repository: controlRepository(access.url), +}); + /** Runs `work` against a restic helper container that is always removed afterwards. */ async function withRestic( docker: DockerApi, - access: ControlRepositoryAccess, + access: ResticAccess, work: ( exec: Exec, run: (cmd: string[], file: string, mode: 'read' | 'write') => Promise, @@ -126,7 +141,7 @@ async function withRestic( ): Promise { const redact = (text: string) => redactSecrets(text, [...Object.values(access.env), access.password]); - const repository = controlRepository(access.url); + const repository = access.repository; const binds = repository.startsWith('/') ? [`${repository}:${repository}`] : []; const limit = access.limitMs ?? CONTROL_JOB_LIMIT_MS; let container: string | undefined; @@ -293,15 +308,14 @@ export async function pushControlArchive( archivePath: string, options: { tags?: string[] } = {}, ): Promise { - return withRestic(docker, access, async (exec, run) => { + return withRestic(docker, controlAccess(access), async (exec, run) => { let repository: ControlPush['repository'] = 'ok'; const probe = await exec(['restic', 'cat', 'config']); - if (probe.code !== 0) { - // Only a missing repository is created; a wrong password or an unreachable bucket must fail the backup. - if (!isMissingRepository(probe.err)) - throw new ControlBackupError( - `repository check failed: ${summarize(probe.err || probe.out)}`, - ); + const state = classifyRepositoryProbe(probe); + if (state !== 'ok') { + // Only a missing repository (restic exit code 10) is created; a wrong password or an unreachable bucket must fail the backup. + if (state === 'error') + throw new ControlBackupError(`repository check failed: ${await explainProbe(exec, probe)}`); const init = await exec(['restic', 'init']); if (init.code !== 0) throw new ControlBackupError(`repository init failed: ${summarize(init.err || init.out)}`); @@ -409,14 +423,14 @@ export async function listControlSnapshots( docker: DockerApi, access: ControlRepositoryAccess, ): Promise<{ state: 'ok' | 'missing'; snapshots: ControlSnapshot[] }> { - return withRestic(docker, access, async (exec) => { + return withRestic(docker, controlAccess(access), async (exec) => { const probe = await exec(['restic', 'cat', 'config']); - if (probe.code !== 0) { - if (isMissingRepository(probe.err)) return { state: 'missing' as const, snapshots: [] }; + const state = classifyRepositoryProbe(probe); + if (state === 'missing') return { state: 'missing' as const, snapshots: [] }; + if (state === 'error') throw new ControlBackupError( - `cannot read the control repository (wrong control password or credentials, or an unreachable store): ${summarize(probe.err || probe.out)}`, + `cannot read the control repository (wrong control password or credentials, or an unreachable store): ${await explainProbe(exec, probe)}`, ); - } const listed = await exec(['restic', 'snapshots', '--json', ...SNAPSHOT_FILTER]); if (listed.code !== 0) throw new ControlBackupError( @@ -437,13 +451,14 @@ export async function fetchControlArchive( throw new ControlBackupError( 'the snapshot must be an id from `tug control backup list` (8 to 64 hex digits)', ); - await withRestic(docker, access, async (exec, run) => { + await withRestic(docker, controlAccess(access), async (exec, run) => { const probe = await exec(['restic', 'cat', 'config']); - if (probe.code !== 0) + const state = classifyRepositoryProbe(probe); + if (state !== 'ok') throw new ControlBackupError( - isMissingRepository(probe.err) + state === 'missing' ? `no control backup was found at ${controlRepository(access.url)}` - : `cannot read the control repository (wrong control password or credentials, or an unreachable store): ${summarize(probe.err || probe.out)}`, + : `cannot read the control repository (wrong control password or credentials, or an unreachable store): ${await explainProbe(exec, probe)}`, ); if (snapshot !== 'latest') { // Only snapshots control itself stored may be restored into control, whatever else the repository holds. @@ -466,3 +481,48 @@ export async function fetchControlArchive( } }); } + +export interface RepositoryCheck { + state: 'ok' | 'initialized' | 'error'; + /** Redacted restic message when `state` is `error`. */ + error?: string; +} + +/** + * Checks one repository the way the first backup would use it: an existing one is `ok`, a missing one (restic exit + * code 10) is initialized, which also proves write access, and anything else is an `error` with restic's message. + * Never throws for a backend problem; the helper's own failures (timeout, cancel, Docker) come back as `error` too. + */ +export async function checkRepository( + docker: DockerApi, + access: Omit & { limitMs?: number }, +): Promise { + try { + return await withRestic(docker, access, async (exec) => { + const probe = await exec(['restic', 'cat', 'config']); + const state = classifyRepositoryProbe(probe); + if (state === 'ok') return { state: 'ok' as const }; + if (state === 'error') + return { + state: 'error' as const, + error: await explainProbe(exec, probe), + }; + const init = await exec(['restic', 'init']); + if (init.code !== 0) + return { + state: 'error' as const, + error: `repository init failed: ${summarize(init.err || init.out)}`, + }; + return { state: 'initialized' as const }; + }); + } catch (error) { + // withRestic redacts what it throws; this repeats it so nothing stored or returned can carry a credential. + return { + state: 'error', + error: redactSecrets(error instanceof Error ? error.message : String(error), [ + ...Object.values(access.env), + access.password, + ]), + }; + } +} diff --git a/src/backup/destination-store.test.ts b/src/backup/destination-store.test.ts index 58726f1..fcb4900 100644 --- a/src/backup/destination-store.test.ts +++ b/src/backup/destination-store.test.ts @@ -152,3 +152,29 @@ test('failed destination removal restores its credentials', async (t) => { 'a failed removal must restore the destination credentials', ); }); + +test('a check result is stored only while the destination is unchanged since it was read, credentials included', async (t) => { + const db = await openDb(':memory:'); + t.after(() => db.close()); + const destination = backupDestinationQueries(db); + const later = '2026-11-03T10:05:00.000Z'; + assert.equal( + destination.recordCheck(at, at, { ok: true }), + false, + 'there is nothing to record a check on before a destination exists', + ); + destination.replace('s3:one', [secret('a', 'A')], at); + assert.equal(destination.recordCheck(at, at, { ok: true }), true, 'the same destination matches'); + assert.deepEqual( + [destination.find()?.checked_at, destination.find()?.check_result_json], + [at, '{"ok":true}'], + 'the check time and result are persisted', + ); + destination.replace('s3:one', [secret('b', 'B')], later); + assert.equal( + destination.recordCheck(at, at, { ok: false }), + false, + 'a result for a destination replaced meanwhile (even with the same URL, new credentials) is dropped', + ); + assert.equal(destination.find()?.check_result_json, null, 'the new destination stays unchecked'); +}); diff --git a/src/backup/destination-store.ts b/src/backup/destination-store.ts index 3532b78..8fd39b8 100644 --- a/src/backup/destination-store.ts +++ b/src/backup/destination-store.ts @@ -44,6 +44,21 @@ export function backupDestinationQueries(db: Db) { ); }); }, + /** + * Stores a check result, but only while the destination is still the one that was checked. `updatedAt` is the + * value read before the check began; every replace sets a new one, so a changed URL or changed credentials drop + * the stale result. False when it was dropped. + */ + recordCheck(updatedAt: string, checkedAt: string, result: unknown): boolean { + return ( + db.run( + "UPDATE backup_destination SET checked_at=?,check_result_json=? WHERE id='default' AND updated_at=?", + checkedAt, + JSON.stringify(result), + updatedAt, + ).changes > 0 + ); + }, remove(): boolean { return immediateTransaction(db, () => { db.run('DELETE FROM secrets WHERE project_id=?', BACKUP_SECRET_PROJECT); diff --git a/src/backup/destination.test.ts b/src/backup/destination.test.ts index 4127347..47f1ee2 100644 --- a/src/backup/destination.test.ts +++ b/src/backup/destination.test.ts @@ -32,6 +32,15 @@ test('invalid destinations are rejected with a message naming the problem', () = [{ url: 's3:https://host/b', env: { ...s3Env, RESTIC_PASSWORD: 'x' } }, /managed by Tug/], [{ url: 's3:https://host/b', env: { ...s3Env, EXTRA: '' } }, /EXTRA needs a value/], [{ url: 's3:https://host/b c', env: s3Env }, /whitespace/], + [ + { url: 'sftp:backup@host:/srv/tug', env: {} }, + /sftp: destinations are not supported.*SSH keys.*rest-server.*S3/, + ], + [{ url: 'rclone:remote:path', env: {} }, /rclone: destinations are not supported/], + [ + { url: 'webdav://host/x', env: {} }, + /url must start with s3:, rest:, b2:, azure:, gs: or swift:/, + ], ]; for (const [input, message] of cases) assert.throws( @@ -63,3 +72,23 @@ test('redaction removes secret values and URL credentials but not short ordinary assert.ok(!redacted.includes('hunter22'), 'a password in a URL should be removed'); assert.ok(redacted.endsWith('in db'), 'values under four characters are left alone'); }); + +test('every built-in restic backend Tug supports is accepted', () => { + for (const url of [ + 'rest:http://backup.internal:8000/tug', + 'b2:bucket:tug', + 'azure:container:/tug', + 'gs:bucket:/tug', + 'swift:container:/tug', + '/srv/restic', + ]) + assert.equal(parseBackupDestination({ url, env: {} }).url, url, `${url} should be accepted`); + assert.equal( + parseBackupDestination({ + url: 'rest:http://backup.internal:8000/tug/', + env: { RESTIC_REST_USERNAME: 'tug', RESTIC_REST_PASSWORD: 'pw-value' }, + }).url, + 'rest:http://backup.internal:8000/tug', + 'a rest url with credentials in env keeps the url normalized', + ); +}); diff --git a/src/backup/destination.ts b/src/backup/destination.ts index 5305dec..472edac 100644 --- a/src/backup/destination.ts +++ b/src/backup/destination.ts @@ -7,6 +7,8 @@ const ENV_NAME_RE = /^[A-Z][A-Z0-9_]{0,63}$/; const RESERVED_ENV = ['RESTIC_PASSWORD', 'RESTIC_REPOSITORY', 'RESTIC_PASSWORD_FILE']; /** Shorter values would redact ordinary words out of error text. */ const MIN_REDACTED_LENGTH = 4; +/** restic's built-in backends Tug accepts; `local` is an absolute path. `sftp:` and `rclone:` are rejected with their own messages. */ +const BACKENDS = ['s3', 'rest', 'b2', 'azure', 'gs', 'swift', 'local']; const REQUIRED_ENV: Record = { s3: ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'], }; @@ -30,9 +32,17 @@ export function parseBackupDestination(input: unknown): BackupDestinationInput { throw new Error('url is required, such as s3:https://s3.example.com/bucket/tug-backups'); if (/\s/.test(url)) throw new Error('url must not contain whitespace'); const scheme = url.startsWith('/') ? 'local' : url.split(':')[0]; - if (!['s3', 'sftp', 'rest', 'local'].includes(scheme)) + if (scheme === 'sftp') throw new Error( - 'url must start with s3:, sftp:, rest: or be an absolute path (a path is for development only: it is read on each node, not shared)', + 'sftp: destinations are not supported because Tug cannot give the restic helper SSH keys or a known_hosts file; run a restic rest-server and reach it over a private network or VPN (rest:http://host:8000/path), or use S3', + ); + if (scheme === 'rclone') + throw new Error( + 'rclone: destinations are not supported because the restic helper image does not configure rclone; use S3, a rest-server (rest:), or another built-in backend', + ); + if (!BACKENDS.includes(scheme)) + throw new Error( + 'url must start with s3:, rest:, b2:, azure:, gs: or swift:, or be an absolute path (a path is for development only: it is read on each node, not shared)', ); if (/:\/\/[^/@\s]*:[^/@\s]*@/.test(url)) throw new Error( diff --git a/src/backup/job.ts b/src/backup/job.ts index 5b4d389..3a885a0 100644 --- a/src/backup/job.ts +++ b/src/backup/job.ts @@ -1,5 +1,6 @@ import { Transform } from 'node:stream'; import { redactSecrets } from './destination.ts'; +import { classifyRepositoryProbe, describeProbeFailure } from './repository-probe.ts'; import { LABEL_JOB, LABEL_PROJECT, @@ -350,20 +351,23 @@ export async function runBackupJob( const version = (await exec(['restic', 'version'])).out.match(/restic (\d+\.\d+\.\d+)/)?.[1]; phase('checking'); - let repository: 'ok' | 'initialized' = 'ok'; + let repository: 'ok' | 'initialized' | 'missing' = 'ok'; const probe = await exec(['restic', 'cat', 'config']); - if (probe.code !== 0) { - // Only a missing repository is initialized; a wrong password or an unreachable bucket must fail the job. - if ( - kind === 'backup.restore' || - !/Is there a repository at the following location|does not exist|no such file/i.test( - probe.err, - ) - ) - throw fail('repository check failed', probe.err || probe.out); - const init = await exec(['restic', 'init']); - if (init.code !== 0) throw fail('repository init failed', init.err || init.out); - repository = 'initialized'; + const state = classifyRepositoryProbe(probe); + if (state === 'error' || (state === 'missing' && kind === 'backup.restore')) + // A restore needs an existing repository; a wrong password or an unreachable bucket fails every kind of job. + throw fail( + 'repository check failed', + describeProbeFailure(probe, probe.err || probe.out, version), + ); + if (state === 'missing') { + // `backup.check` only reports; `backup.run` creates the repository, as only restic exit code 10 means it is absent. + if (kind === 'backup.check') repository = 'missing'; + else { + const init = await exec(['restic', 'init']); + if (init.code !== 0) throw fail('repository init failed', init.err || init.out); + repository = 'initialized'; + } } if (kind === 'backup.check') return { diff --git a/src/backup/repository-probe.test.ts b/src/backup/repository-probe.test.ts new file mode 100644 index 0000000..d57d77b --- /dev/null +++ b/src/backup/repository-probe.test.ts @@ -0,0 +1,71 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + RESTIC_EXIT_REPOSITORY_MISSING, + RESTIC_EXIT_WRONG_PASSWORD, + classifyRepositoryProbe, + resticTooOld, + describeProbeFailure, +} from './repository-probe.ts'; + +const probe = (code: number | null, err = '', out = '') => ({ code, err, out }); + +test('restic exit codes 0, 10 and anything else are ok, missing and error', () => { + assert.equal(RESTIC_EXIT_REPOSITORY_MISSING, 10, 'restic documents 10 as repository missing'); + assert.equal(RESTIC_EXIT_WRONG_PASSWORD, 12, 'restic documents 12 as wrong password'); + assert.equal(classifyRepositoryProbe(probe(0)), 'ok', 'exit 0 is an existing repository'); + assert.equal(classifyRepositoryProbe(probe(10)), 'missing', 'exit 10 is a missing repository'); + for (const code of [1, 11, 12, 130, null]) + assert.equal(classifyRepositoryProbe(probe(code)), 'error', `exit ${code} is an error`); +}); + +test('the message wording never decides whether a repository is missing', () => { + assert.equal( + classifyRepositoryProbe( + probe( + 1, + 'The specified bucket does not exist\nIs there a repository at the following location?', + ), + ), + 'error', + 'a missing bucket with exit 1 is an error even though it says "does not exist"', + ); + assert.equal( + classifyRepositoryProbe(probe(10, 'anything at all')), + 'missing', + 'exit 10 is missing whatever the text says', + ); +}); + +test('failures get the hint that fixes them', () => { + assert.match( + describeProbeFailure(probe(12, 'Fatal: wrong password'), 'Fatal: wrong password'), + /wrong repository password/, + 'exit 12 reads as a wrong password', + ); + const hinted = + 'Fatal: The specified bucket does not exist\nIs there a repository at the following location?'; + assert.doesNotMatch( + describeProbeFailure(probe(1, hinted), hinted, '0.19.1'), + /older than 0\.17/, + 'restic 0.19 appends the repository hint to ordinary failures, so a mistyped bucket must not blame the version', + ); + assert.doesNotMatch( + describeProbeFailure(probe(1, hinted), hinted), + /older than 0\.17/, + 'an unknown version gets no hint', + ); + assert.match( + describeProbeFailure(probe(1, hinted), hinted, '0.16.4'), + /restic 0\.16\.4 is older than 0\.17.*TUG_RESTIC_IMAGE must be 0\.17 or newer/, + 'a known restic older than 0.17 gets the hint', + ); + assert.equal(resticTooOld('0.17.0'), false, '0.17.0 is new enough'); + assert.equal(resticTooOld('1.0.0'), false, 'a later major is new enough'); + assert.equal(resticTooOld('0.9.6'), true, '0.9.6 is too old'); + assert.equal( + describeProbeFailure(probe(1, 'dial tcp: no such host'), 'dial tcp: no such host'), + 'dial tcp: no such host', + 'other failures keep restic message unchanged', + ); +}); diff --git a/src/backup/repository-probe.ts b/src/backup/repository-probe.ts new file mode 100644 index 0000000..425c5f3 --- /dev/null +++ b/src/backup/repository-probe.ts @@ -0,0 +1,58 @@ +/** restic (0.17 and newer) exits with this code when the repository does not exist. */ +export const RESTIC_EXIT_REPOSITORY_MISSING = 10; +/** restic (0.17 and newer) exits with this code when the repository password is wrong. */ +export const RESTIC_EXIT_WRONG_PASSWORD = 12; + +export type RepositoryProbe = 'ok' | 'missing' | 'error'; + +export interface ProbeResult { + code: number | null; + err: string; + out: string; +} + +/** + * Classifies `restic cat config` by its exit code, the one signal restic documents for "no repository here": 0 is an + * existing repository, 10 a missing one and anything else (a wrong password, a missing bucket, a refused login, an + * unreachable store, a restic too old to report 10) is an error. Wording is never matched: a typo in an S3 bucket + * name also says "does not exist" but must not make Tug create the bucket. + */ +export function classifyRepositoryProbe(result: ProbeResult): RepositoryProbe { + if (result.code === 0) return 'ok'; + if (result.code === RESTIC_EXIT_REPOSITORY_MISSING) return 'missing'; + return 'error'; +} + +/** The first restic that exits with code 10 for a missing repository. */ +export const MIN_RESTIC_VERSION = [0, 17, 0] as const; + +/** True when `version` (`0.16.4`) is known and older than {@link MIN_RESTIC_VERSION}. */ +export function resticTooOld(version: string | undefined): boolean { + const parts = version + ?.match(/^(\d+)\.(\d+)\.(\d+)/) + ?.slice(1) + .map(Number); + if (!parts) return false; + for (let i = 0; i < 3; i++) { + if (parts[i] !== MIN_RESTIC_VERSION[i]) return parts[i] < MIN_RESTIC_VERSION[i]; + } + return false; +} + +/** + * Explains a failed probe in words. `detail` is restic's own (already summarized and redacted) message, which this + * only prefixes or suffixes. A wrong password (exit 12) is named. The old-restic hint needs the restic version: the + * message text cannot tell it, because restic 0.19 appends "Is there a repository at the following location?" to + * every exit-1 failure too (a mistyped bucket, a refused login). + */ +export function describeProbeFailure( + result: ProbeResult, + detail: string, + resticVersion?: string, +): string { + if (result.code === RESTIC_EXIT_WRONG_PASSWORD) + return `wrong repository password (restic exit code ${RESTIC_EXIT_WRONG_PASSWORD}): ${detail}`; + if (result.code === 1 && resticTooOld(resticVersion)) + return `${detail} (restic ${resticVersion} is older than 0.17 and cannot report a missing repository, so TUG_RESTIC_IMAGE must be 0.17 or newer for repositories to be created automatically)`; + return detail; +} diff --git a/src/cli/commands/backup.ts b/src/cli/commands/backup.ts index 4fe28d9..eb4f7c0 100644 --- a/src/cli/commands/backup.ts +++ b/src/cli/commands/backup.ts @@ -44,6 +44,52 @@ interface BackupListing { }[]; runs: BackupRunRecord[]; } +interface DestinationCheck { + ok: boolean; + checkedAt: string; + from: 'control'; + url: string; + repositories: { + name: string; + repository: string; + state: 'ok' | 'initialized' | 'error' | 'skipped'; + error?: string; + }[]; +} +/** Where the check runs, so a pass is not read as proof that every node can reach the destination. */ +const CHECK_SCOPE = + 'Checked from the control host; each node connects to the destination at its first backup.'; +/** One line per repository on stderr, saying what the state means for the operator. */ +function reportCheck(rt: CliRuntime, check: DestinationCheck): void { + for (const r of check.repositories) { + if (r.state === 'ok') err(rt, `ok: ${r.name} (${r.repository})`); + else if (r.state === 'initialized') + err( + rt, + `created a new, empty repository at ${r.repository} (${r.name}); if backups should already be there, the destination path is wrong`, + ); + else if (r.state === 'skipped') err(rt, `skipped: ${r.name}: ${r.error ?? 'not checked'}`); + else { + err(rt, `failed: ${r.name} (${r.repository}): ${r.error ?? 'failed'}`); + err( + rt, + r.name === '_control' + ? " control's own nightly backup will fail" + : ' failed from the control host; nodes connect to the destination themselves at their first backup, so check that path if control and nodes are on different networks', + ); + } + } + err(rt, CHECK_SCOPE); +} +const failedNames = (check: DestinationCheck): string => + check.repositories + .filter((r) => r.state === 'error') + .map((r) => r.name) + .join(', '); +const checkDestination = (rt: CliRuntime): Promise => + withProgress(rt, 'Checking the backup destination', () => + request(rt, 'POST', '/api/v1/backup/destination/check', {}), + ); export const formatBytes = (bytes: number | null): string => bytes === null ? '-' @@ -224,14 +270,17 @@ export function registerBackup(rt: CliRuntime, program: Command): void { if (o.control) return print(rt, await request(rt, 'GET', '/api/v1/backup/repository-password?control=1')); const project = loadCfg(rt).project || 'site'; - print( + const result = await request<{ deployed?: boolean }>( rt, - await request( - rt, - 'GET', - `/api/v1/backup/repository-password?project=${encodeURIComponent(project)}`, - ), + 'GET', + `/api/v1/backup/repository-password?project=${encodeURIComponent(project)}`, ); + print(rt, result); + if (result.deployed === false) + err( + rt, + `project ${project} has not been deployed yet: check the project name, because the password is derived from it`, + ); }); const destination = backup @@ -240,7 +289,7 @@ export function registerBackup(rt: CliRuntime, program: Command): void { destination .command('set ') .description( - 'set where backups go, such as s3:https://s3.example.com/bucket/tug-backups; credentials are stored encrypted on control', + 'set where backups go, such as s3:https://s3.example.com/bucket/tug-backups, rest:http://backup.internal:8000/tug or b2:bucket:tug; credentials are stored encrypted on control. Then checks the destination from the control host (creating missing repositories); nodes connect at their first backup. A failed check is a warning and the exit code stays 0, since the destination is saved; --no-check skips it', ) .option( '--env ', @@ -248,16 +297,56 @@ export function registerBackup(rt: CliRuntime, program: Command): void { collectList, [], ) - .action(async (url: string, o: { env: string[] }) => { + .option('--no-check', 'save the destination without checking it') + .action(async (url: string, o: { env: string[]; check: boolean }) => { const env = await readEnvValues(rt, o.env); - out( + const saved = await request>( rt, - JSON.stringify(await request(rt, 'POST', '/api/v1/backup/destination', { url, env })), + 'POST', + '/api/v1/backup/destination', + { url, env }, ); + // Saving succeeded, so nothing below changes the exit code: a failed or unavailable check is a warning. + let check: DestinationCheck | undefined; + let unavailable = false; + if (o.check) + try { + check = await checkDestination(rt); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/destination\/check: 404/.test(message)) throw error; + unavailable = true; + } + // One JSON document, so scripts that parse the output of `set` keep working. + out(rt, JSON.stringify(check ? { ...saved, check } : saved)); err( rt, 'Control now backs itself up here every night. Keep its password with these credentials: tug backup password --control', ); + if (unavailable) + err( + rt, + 'saved; this control is too old to check destinations (it has no destination check), so nothing was verified. Upgrade control to use tug backup destination check', + ); + if (check) { + reportCheck(rt, check); + if (!check.ok) + err( + rt, + `saved; the check failed for ${failedNames(check) || 'some repositories'}: fix it and run tug backup destination check`, + ); + } + }); + destination + .command('check') + .description( + 'check the destination from the control host: open the control repository and the repository of every project with a backup target, and create the ones that do not exist yet (which also proves write access). Exits nonzero when any repository cannot be used. Nodes connect to the destination at their first backup, so this does not prove every node can reach it', + ) + .action(async () => { + const check = await checkDestination(rt); + print(rt, check); + reportCheck(rt, check); + if (!check.ok) rt.exitCode = 1; }); destination .command('show') diff --git a/src/cli/credentials.test.ts b/src/cli/credentials.test.ts index 96812e9..8ed1426 100644 --- a/src/cli/credentials.test.ts +++ b/src/cli/credentials.test.ts @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import { startMtlsControl } from '../../tests/helpers.ts'; import { execCli } from '../../tests/cli-inprocess.ts'; +import { FakeDocker } from '../../tests/fake-docker.ts'; const OPERATOR = 'op-cli-tokens'; @@ -190,7 +191,7 @@ test('backup commands are listed in help', async () => { (await execCli([...group, '--help'], { env, cwd: dir })).stdout; for (const [group, verbs] of [ ['backup', ['run', 'list', 'password', 'destination']], - ['backup destination', ['set', 'show', 'remove']], + ['backup destination', ['set', 'show', 'check', 'remove']], ['restore', ['run', 'list', 'resume', 'release']], ] as const) { const help = await groupHelp(...group.split(' ')); @@ -221,6 +222,7 @@ test('backup destination commands set, show and remove the destination without e 'destination', 'set', 's3:https://s3.example.com/bucket/tug', + '--no-check', '--env', 'AWS_ACCESS_KEY_ID', '--env', @@ -231,7 +233,7 @@ test('backup destination commands set, show and remove the destination without e assert.equal(shown.url, 's3:https://s3.example.com/bucket/tug', 'show prints the url'); assert.deepEqual(shown.envNames, ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']); await assert.rejects( - run('backup', 'destination', 'set', 's3:https://s3.example.com/bucket/tug'), + run('backup', 'destination', 'set', 's3:https://s3.example.com/bucket/tug', '--no-check'), /need AWS_ACCESS_KEY_ID/, 'a destination without its credentials is refused by control', ); @@ -241,3 +243,103 @@ test('backup destination commands set, show and remove the destination without e await control.close(); } }); + +test('backup destination set checks the destination and check exits nonzero while a repository is unusable', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'tug-backup-check-cli-')); + const docker = new FakeDocker(); + docker.addImage('restic/restic:0.19.1'); + let code = 0; + docker.execResponder = (cmd) => + cmd[1] === 'cat' + ? { + code, + err: + code === 10 + ? 'Fatal: repository does not exist: unable to open config file: does not exist\nIs there a repository at the following location?' + : 'Fatal: unable to open config file: Stat: The specified bucket does not exist.', + } + : { code: 0 }; + const control = await startMtlsControl(dir, { docker }); + const env = { + ...process.env, + ...control.env, + AWS_ACCESS_KEY_ID: 'AKIAEXAMPLE', + AWS_SECRET_ACCESS_KEY: 'cli-secret-value', + }; + delete (env as Record).TUG_CONFIG; + const run = (...args: string[]) => execCli([...args], { env, cwd: dir }); + const setArgs = [ + 'backup', + 'destination', + 'set', + 's3:https://s3.example.com/bucket/tug', + '--env', + 'AWS_ACCESS_KEY_ID', + '--env', + 'AWS_SECRET_ACCESS_KEY', + ]; + try { + code = 10; + const created = await run(...setArgs); + const body = JSON.parse(created.stdout); + assert.equal(body.ok, true, 'set keeps printing the saved destination as one JSON document'); + assert.equal( + body.check.repositories[0].state, + 'initialized', + 'a missing repository is created', + ); + assert.match(created.stderr, /control host/, 'the output says where the check ran'); + + assert.match( + created.stderr, + /created a new, empty repository at .*_control/, + 'a repository that was just created is not reported as plain success', + ); + + code = 1; + const failed = await run(...setArgs); + assert.equal( + JSON.parse(failed.stdout).check.ok, + false, + 'a failed check still exits 0 after a successful save and is reported in the JSON', + ); + assert.match( + failed.stderr, + /saved; the check failed for _control.*tug backup destination check/, + ); + assert.match( + failed.stderr, + /control's own nightly backup will fail/, + 'the stderr says what it means', + ); + assert.match( + failed.stderr, + /Checked from the control host/, + 'the stderr says where the check ran', + ); + const shown = JSON.parse((await run('backup', 'destination', 'show')).stdout); + assert.equal(shown.checkResult.ok, false, 'the failed check is stored with the destination'); + + await assert.rejects( + run('backup', 'destination', 'check'), + (error: Error & { code?: number }) => error.code !== 0, + 'check exits nonzero while a repository fails', + ); + const password = await run('backup', 'password'); + assert.equal( + JSON.parse(password.stdout).deployed, + false, + 'the password is printed before a deploy', + ); + assert.match( + password.stderr, + /has not been deployed yet: check the project name/, + 'an undeployed project gets a warning that the name feeds the password', + ); + code = 0; + const ok = await run('backup', 'destination', 'check'); + assert.equal(JSON.parse(ok.stdout).ok, true, 'check prints a passing result and exits zero'); + } finally { + await control.close(); + } +}); diff --git a/src/cli/stdin.process.test.ts b/src/cli/stdin.process.test.ts index 0b2ea28..3a322c5 100644 --- a/src/cli/stdin.process.test.ts +++ b/src/cli/stdin.process.test.ts @@ -67,6 +67,7 @@ test('whole-document stdin commands and HTTP commands exit after completing thei 'destination', 'set', 's3:https://s3.example.com/bucket/tug', + '--no-check', '--env', 'AWS_ACCESS_KEY_ID', '--env', diff --git a/src/control/backup-destination.test.ts b/src/control/backup-destination.test.ts index 0d80df6..0865502 100644 --- a/src/control/backup-destination.test.ts +++ b/src/control/backup-destination.test.ts @@ -4,20 +4,25 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { createApp } from './control.ts'; +import { FakeDocker } from '../../tests/fake-docker.ts'; +import { controlRepositoryPassword } from '../backup/control-backup.ts'; import { openDb } from '../db/db.ts'; import { decrypt } from './secrets.ts'; +import { until } from '../../tests/wait.ts'; import { json, testImageResolver, testControlUrl, testIdentity } from '../../tests/helpers.ts'; import { repositoryPassword } from '../backup/destination.ts'; +import type { CreateContainerBody } from '../docker-api.ts'; import type { Server } from 'node:http'; import type { AddressInfo } from 'node:net'; -async function server() { +async function server(docker?: FakeDocker) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'tug-bkdest-')); process.env.TUG_DATA_DIR = dir; const app = await createApp({ identity: testIdentity, controlUrl: testControlUrl, images: { resolve: testImageResolver }, + docker, }); const srv = await new Promise((resolve) => { const s = app.listen(0, () => resolve(s)); @@ -137,10 +142,20 @@ test('destination and key endpoints are admin only', async () => { test('the project repository password is derived from the master key and audited', async () => { const s = await server(); try { + const early = await json( + await call(s.base, 'GET', '/api/v1/backup/repository-password?project=ghost'), + ); + const earlyKey = (await fs.readFile(s.keyFile, 'utf8')).trim(); assert.equal( - (await call(s.base, 'GET', '/api/v1/backup/repository-password?project=ghost')).status, - 404, - 'an unknown project has no key', + early.password, + repositoryPassword(earlyKey, 'ghost'), + 'a project that was never deployed still has its derived password', + ); + assert.equal(early.deployed, false, 'the response says the project has not been deployed'); + assert.equal( + (await call(s.base, 'GET', '/api/v1/backup/repository-password?project=_control')).status, + 400, + 'a reserved name has no project password', ); await call(s.base, 'POST', '/api/v1/projects', { name: 'site' }); await call(s.base, 'POST', '/api/v1/backup/destination', destination); @@ -149,6 +164,7 @@ test('the project repository password is derived from the master key and audited ); const key = (await fs.readFile(s.keyFile, 'utf8')).trim(); assert.equal(body.password, repositoryPassword(key, 'site'), 'the password is derived'); + assert.equal(body.deployed, true, 'an existing project is reported as deployed'); assert.equal( body.repository, 's3:https://s3.example.com/bucket/tug/site', @@ -157,7 +173,7 @@ test('the project repository password is derived from the master key and audited const db = await openDb(s.dbFile); assert.equal( db.all("SELECT 1 FROM events WHERE type='backup.key_read'").length, - 1, + 2, 'reading the key is audited', ); } finally { @@ -277,3 +293,231 @@ test('credentials are released only to the owning node for a running backup comm await s.close(); } }); + +/** Remembers every helper container's settings; checks run one helper at a time, so the last one is the one in use. */ +class RecordingDocker extends FakeDocker { + bodies: CreateContainerBody[] = []; + async createContainer(name: string, body: CreateContainerBody): Promise { + this.bodies.push(body); + return super.createContainer(name, body); + } +} +const envOf = (body: CreateContainerBody | undefined, name: string): string | undefined => + body?.Env?.find((entry) => entry.startsWith(`${name}=`))?.slice(name.length + 1); +const composeWithBackup = { + 'x-tug': { backup: { schedule: '0 3 * * *', retention: { daily: 7 } } }, + services: { + db: { + image: 'mariadb:11.4', + volumes: [{ type: 'volume', source: 'db-data', target: '/var/lib/mysql' }], + 'x-tug': { placement: { nodes: ['n1'] }, backup: { command: 'dump-it', verify: 'done' } }, + }, + }, + volumes: { 'db-data': {} }, +}; +const REST_MISSING = + 'Fatal: repository does not exist: unable to open config file: does not exist\nIs there a repository at the following location?'; + +async function deployWithBackup(base: string): Promise { + const node = await json(await call(base, 'POST', '/api/v1/nodes/register', { name: 'n1' })); + await call(base, 'POST', '/api/v1/projects', { name: 'empty' }); + const deploy = await call(base, 'POST', '/api/v1/deployments', { + project: 'site', + version: 'v1', + compose: composeWithBackup, + }); + assert.equal(deploy.status, 200, await deploy.clone().text()); + const agent = { 'x-tug-test-node': node.nodeId, 'content-type': 'application/json' }; + const work = await json(await call(base, 'GET', '/agent/v1/work', undefined, agent)); + await call( + base, + 'POST', + '/agent/v1/status', + { + projects: [ + { + project: 'site', + observedVersion: 'v1', + observedRevision: work.desiredStates[0].revision, + health: 'serving', + }, + ], + commandResults: [], + }, + agent, + ); +} + +test('a destination check opens the control and project repositories, creates missing ones and stores the result', async () => { + const docker = new RecordingDocker(); + docker.addImage('restic/restic:0.19.1'); + const missing = new Set(); + docker.execResponder = (cmd) => { + const repository = envOf(docker.bodies.at(-1), 'RESTIC_REPOSITORY') ?? ''; + return cmd[1] === 'cat' && missing.has(repository) + ? { code: 10, err: REST_MISSING } + : { code: 0 }; + }; + const s = await server(docker); + try { + assert.equal( + (await call(s.base, 'POST', '/api/v1/backup/destination/check', {})).status, + 409, + 'a check without a destination is refused with a reason', + ); + await deployWithBackup(s.base); + await call(s.base, 'POST', '/api/v1/backup/destination', { + url: 'rest:http://backup.internal:8030/tug', + env: { RESTIC_REST_USERNAME: 'tug', RESTIC_REST_PASSWORD: 'rest-secret-1' }, + }); + missing.add('rest:http://backup.internal:8030/tug/_control'); + const res = await call(s.base, 'POST', '/api/v1/backup/destination/check', {}); + assert.equal(res.status, 200, await res.clone().text()); + const result = await json(res); + assert.equal(result.ok, true, 'every repository is usable'); + assert.equal(result.from, 'control', 'the result says where it ran'); + assert.equal(result.url, 'rest:http://backup.internal:8030/tug', 'the checked url is echoed'); + assert.deepEqual( + result.repositories.map((r: { name: string; state: string }) => [r.name, r.state]), + [ + ['_control', 'initialized'], + ['site', 'ok'], + ], + 'control and each project with a backup target are checked; a missing repository is created; a project without targets is skipped', + ); + const key = (await fs.readFile(s.keyFile, 'utf8')).trim(); + const passwords = docker.bodies.map((b) => envOf(b, 'RESTIC_PASSWORD')); + assert.deepEqual( + passwords, + [controlRepositoryPassword(key), repositoryPassword(key, 'site')], + 'each repository is opened with its own derived password', + ); + const shown = await json(await call(s.base, 'GET', '/api/v1/backup/destination')); + assert.equal(shown.checkResult.ok, true, 'the result is stored with the destination'); + assert.equal(shown.checkedAt, result.checkedAt, 'the check time is stored'); + const db = await openDb(s.dbFile); + assert.equal( + db.all("SELECT 1 FROM events WHERE type='backup.destination_checked'").length, + 1, + 'a check is audited', + ); + assert.ok(!JSON.stringify(result).includes('rest-secret-1'), 'credentials never appear'); + } finally { + await s.close(); + } +}); + +test('a destination check reports a failing repository with ok false and a redacted message', async () => { + const docker = new FakeDocker(); + docker.addImage('restic/restic:0.19.1'); + docker.execResponder = (cmd) => + cmd[1] === 'cat' + ? { + code: 1, + err: 'Fatal: unable to open config file: The specified bucket does not exist for sekret-value-1', + } + : { code: 0 }; + const s = await server(docker); + try { + await call(s.base, 'POST', '/api/v1/backup/destination', destination); + const res = await call(s.base, 'POST', '/api/v1/backup/destination/check', {}); + assert.equal(res.status, 200, 'a failing repository is a result, not an HTTP error'); + const text = await res.text(); + const result = JSON.parse(text); + assert.equal(result.ok, false, 'the check fails'); + assert.equal(result.repositories[0].state, 'error', 'the repository is an error'); + assert.match(result.repositories[0].error, /bucket does not exist/, 'restic message is shown'); + assert.ok(!text.includes('sekret-value-1'), 'the secret is redacted'); + assert.ok( + !docker.execOptions.some((e) => e.cmd[1] === 'init'), + 'a missing bucket never triggers init', + ); + const shown = await json(await call(s.base, 'GET', '/api/v1/backup/destination')); + assert.equal(shown.checkResult.ok, false, 'the failure is stored'); + } finally { + await s.close(); + } +}); + +test('a manual backup of a project that was never deployed says it has not been deployed yet', async () => { + const s = await server(); + try { + const res = await call(s.base, 'POST', '/api/v1/backups/run', { project: 'ghost' }); + assert.equal(res.status, 404, 'an unknown project cannot be backed up'); + assert.match( + (await json(res)).error, + /has not been deployed yet.*backup targets come from its deployed Compose file/, + 'the message says to deploy first', + ); + } finally { + await s.close(); + } +}); + +test('a failing control repository skips the project repositories instead of timing out on each', async () => { + const docker = new RecordingDocker(); + docker.addImage('restic/restic:0.19.1'); + docker.execResponder = (cmd) => + cmd[1] === 'cat' + ? { code: 1, err: 'Fatal: unable to open config file: bucket does not exist' } + : { code: 0 }; + const s = await server(docker); + try { + await deployWithBackup(s.base); + await call(s.base, 'POST', '/api/v1/backup/destination', destination); + const result = await json(await call(s.base, 'POST', '/api/v1/backup/destination/check', {})); + assert.deepEqual( + result.repositories.map((r: { name: string; state: string }) => [r.name, r.state]), + [ + ['_control', 'error'], + ['site', 'skipped'], + ], + 'the project repository is skipped once control fails on the same destination', + ); + assert.match(result.repositories[1].error, /not checked: control's own repository failed/); + assert.equal(result.ok, false, 'a skipped repository still fails the check'); + assert.equal( + docker.bodies.length, + 1, + 'no helper container is started for the skipped repository', + ); + } finally { + await s.close(); + } +}); + +test('a second concurrent check is refused and a destination replaced mid-check keeps no stale result', async () => { + let release: () => void = () => {}; + const gate = new Promise((resolve) => (release = resolve)); + class GatedDocker extends RecordingDocker { + async createContainer(name: string, body: CreateContainerBody): Promise { + const id = await super.createContainer(name, body); + await gate; + return id; + } + } + const docker = new GatedDocker(); + docker.addImage('restic/restic:0.19.1'); + docker.execResponder = () => ({ code: 0 }); + const s = await server(docker); + try { + await call(s.base, 'POST', '/api/v1/backup/destination', destination); + const first = call(s.base, 'POST', '/api/v1/backup/destination/check', {}); + await until(() => docker.bodies.length === 1, 'the first check to start its helper'); + const second = await call(s.base, 'POST', '/api/v1/backup/destination/check', {}); + assert.equal(second.status, 409, 'a check while another runs is refused'); + assert.match((await json(second)).error, /already running/, 'the reason is given'); + await call(s.base, 'POST', '/api/v1/backup/destination', { + ...destination, + env: { ...destination.env, AWS_SECRET_ACCESS_KEY: 'rotated-secret-2' }, + }); + release(); + assert.equal((await first).status, 200, 'the running check still answers'); + const shown = await json(await call(s.base, 'GET', '/api/v1/backup/destination')); + assert.equal(shown.checkResult, null, 'a result for the replaced credentials is not stored'); + assert.equal(shown.checkedAt, null, 'the replaced destination stays unchecked'); + } finally { + release(); + await s.close(); + } +}); diff --git a/src/control/control-backup.test.ts b/src/control/control-backup.test.ts index 52895a8..88ca125 100644 --- a/src/control/control-backup.test.ts +++ b/src/control/control-backup.test.ts @@ -335,8 +335,8 @@ test('restore from the destination reports an empty destination without touching const docker = resticDocker((cmd) => cmd[1] === 'cat' ? { - code: 1, - err: 'Fatal: unable to open config file: Stat: The specified key does not exist.\nIs there a repository at the following location?', + code: 10, + err: 'Fatal: repository does not exist: unable to open config file: Stat: The specified key does not exist.\nIs there a repository at the following location?', } : { code: 0 }, ); diff --git a/src/control/control.ts b/src/control/control.ts index 9ced8bd..7caf31a 100755 --- a/src/control/control.ts +++ b/src/control/control.ts @@ -137,6 +137,7 @@ import { CONTROL_SNAPSHOT_ID, controlRepository, controlRepositoryPassword, + checkRepository, fetchControlArchive, listControlSnapshots, pushControlArchive, @@ -297,6 +298,10 @@ const liveDimension = (value: unknown) => const liveBase64 = (value: unknown): value is string => typeof value === 'string' && /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value); +/** One repository of a destination check, including the image pull, may take this long. */ +const DESTINATION_CHECK_REPOSITORY_LIMIT_MS = 2 * 60_000; +/** The whole destination check (every repository) stops after this long; the rest are reported as errors. */ +const DESTINATION_CHECK_LIMIT_MS = 4 * 60_000; /** How long a host may lack a certificate, pending or failing its first attempts, before `tug doctor` fails: DNS and port 80 can lag a new domain. */ const PENDING_TLS_GRACE_MS = 10 * 60_000; /** How long `doctor` waits for a TCP connect to an ingress node's port 80 or 443 before calling it unanswered. */ @@ -2311,7 +2316,9 @@ export async function createApp( ) return res.status(400).json({ error: 'targets must be a list of service or volume names' }); if (!db.get('SELECT 1 FROM projects WHERE id=?', project)) - return res.status(404).json({ error: `project ${project} not found` }); + return res.status(404).json({ + error: `project ${project} has not been deployed yet; backup targets come from its deployed Compose file, so deploy it first`, + }); const result = backups.runNow(project, targets, req.actor!.name); if (!result.runs.length) return res.status(result.errors.length ? 409 : 404).json({ @@ -2455,8 +2462,8 @@ export async function createApp( const project = String(req.query.project || ''); if (!PROJECT_NAME_PATTERN.test(project) || isReservedProject(project)) return res.status(400).json({ error: 'project is required' }); - if (!db.get('SELECT 1 FROM projects WHERE id=?', project)) - return res.status(404).json({ error: `project ${project} not found` }); + // The password is derived from the master key and the name alone, so it exists before the project is deployed. + const deployed = Boolean(db.get('SELECT 1 FROM projects WHERE id=?', project)); const destination = backupDestination(); audit(req.actor!, { type: 'backup.key_read', @@ -2467,8 +2474,99 @@ export async function createApp( project, repository: destination ? projectRepository(destination.url, project) : null, password: repositoryPassword(masterKey, project), + deployed, }); }); + /** One check at a time: it holds a helper container per repository and may create repositories. */ + let destinationCheckRunning = false; + api('post', '/api/v1/backup/destination/check', null, async (req, res) => { + // The URL and its credentials are read together, before anything awaits. + const destination = backupDestination(); + if (!destination) + return res.status(409).json({ + error: 'no backup destination is configured; set one with tug backup destination set', + }); + if (destinationCheckRunning) + return res.status(409).json({ error: 'a destination check is already running' }); + if (controlPushRunning) + return res.status(409).json({ + error: 'control is backing itself up to the destination right now; try again in a minute', + }); + // Matches the stored check to this exact destination (URL and credentials) if it was replaced meanwhile. + const checkedVersion = destination.updated_at; + const env = backupEnv(); + backups.sync(); + const projects = db + .all<{ project_id: string }>( + 'SELECT DISTINCT project_id FROM backup_schedules WHERE enabled=1 ORDER BY project_id', + ) + .map((row) => row.project_id); + destinationCheckRunning = true; + try { + const overall = AbortSignal.timeout(DESTINATION_CHECK_LIMIT_MS); + const repositories: { + name: string; + repository: string; + state: 'ok' | 'initialized' | 'error' | 'skipped'; + error?: string; + }[] = []; + const targets = [ + { + name: '_control', + repository: controlRepository(destination.url), + password: controlRepositoryPassword(masterKey), + }, + ...projects.map((project) => ({ + name: project, + repository: projectRepository(destination.url, project), + password: repositoryPassword(masterKey, project), + })), + ]; + for (const target of targets) { + // Every repository shares the destination, so when control's own fails the others would fail the same way, + // each after its own timeout, and the request would outlast the CLI's. + const controlFailed = repositories[0]?.state === 'error'; + const checked = controlFailed + ? { + state: 'skipped' as const, + error: "not checked: control's own repository failed on the same destination", + } + : overall.aborted + ? { state: 'error' as const, error: 'the destination check ran out of time' } + : await checkRepository(docker, { + repository: target.repository, + env, + password: target.password, + resticImage: resticImage(), + limitMs: DESTINATION_CHECK_REPOSITORY_LIMIT_MS, + stallMs: controlStallMs(), + signal: overall, + }); + repositories.push({ name: target.name, repository: target.repository, ...checked }); + } + const result = { + ok: repositories.every((repository) => repository.state !== 'error'), + checkedAt: now(), + from: 'control' as const, + url: destination.url, + repositories, + }; + const stored = destinationQueries.recordCheck(checkedVersion, result.checkedAt, result); + audit(req.actor!, { + type: 'backup.destination_checked', + message: `backup destination checked by ${req.actor!.name}: ${result.ok ? 'ok' : 'failed'}`, + metadata: { + url: destination.url, + ok: result.ok, + repositories: repositories.map(({ name, state }) => ({ name, state })), + stored, + }, + }); + res.json(result); + } finally { + destinationCheckRunning = false; + } + }); api('post', '/api/v1/secrets', bodyProject, async (req, res) => { const { project = 'site', name, value } = req.body; @@ -4001,7 +4099,8 @@ export async function createApp( // The URL and its credentials are read together, before anything awaits: a destination changed meanwhile // must not pair the old URL with the new credentials. const destination = backupDestination(); - if (!destination || controlPushRunning) return null; + // A destination check is creating or opening the same repositories; the nightly retry follows soon after. + if (!destination || controlPushRunning || destinationCheckRunning) return null; const env = backupEnv(); const key = destinationKey(destination.url); controlPushRunning = true; diff --git a/src/control/credentials.test.ts b/src/control/credentials.test.ts index 0c4dac0..3dc0c50 100644 --- a/src/control/credentials.test.ts +++ b/src/control/credentials.test.ts @@ -250,6 +250,8 @@ const SAMPLES: Record = { body: { url: '/srv/restic', env: {} }, }, 'DELETE /api/v1/backup/destination': { path: '/api/v1/backup/destination' }, + // Listed after the delete, so no destination exists and an admin gets a fast 409 instead of a check on a real Docker. + 'POST /api/v1/backup/destination/check': { path: '/api/v1/backup/destination/check', body: {} }, 'GET /api/v1/backup/repository-password': { path: '/api/v1/backup/repository-password?project=site', }, diff --git a/src/control/credentials.ts b/src/control/credentials.ts index ca3360d..9235bd2 100644 --- a/src/control/credentials.ts +++ b/src/control/credentials.ts @@ -78,6 +78,7 @@ export const ROUTE_ROLES: Record = { 'GET /api/v1/backup/destination': 'admin', 'POST /api/v1/backup/destination': 'admin', 'DELETE /api/v1/backup/destination': 'admin', + 'POST /api/v1/backup/destination/check': 'admin', 'GET /api/v1/backup/repository-password': 'admin', 'POST /api/v1/control/backups/run': 'admin', 'GET /api/v1/control/backups': 'admin', diff --git a/src/protocol.ts b/src/protocol.ts index cd9133a..91c0819 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -279,8 +279,8 @@ export interface BackupJobResult { totalBytes?: number; durationMs: number; resticVersion?: string; - /** Repository state for `backup.check`. */ - repository?: 'ok' | 'initialized'; + /** Repository state: `missing` only from the read-only `backup.check`; `backup.run` initializes instead. */ + repository?: 'ok' | 'initialized' | 'missing'; warnings?: string[]; } diff --git a/tests/helpers.ts b/tests/helpers.ts index 1745d30..bc44cc5 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -148,7 +148,10 @@ export async function writeCredentialFile( */ export async function startMtlsControl( dir: string, - options: { eventPruneIntervalMs?: number } = {}, + options: { + eventPruneIntervalMs?: number; + docker?: import('../src/docker-api.ts').DockerApi; + } = {}, ) { const { createApp, listen } = await import('../src/control/control.ts'); const { issueHostCredential } = await import('../src/control/credential-issue.ts');