diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78f42c3d..4ccf2bff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,20 @@ jobs: GOTOOLCHAIN=auto go test -c -o /tmp/runtime-network.test ./internal/lxc sudo env LXC2DOCKER_NETWORK_TEST=1 unshare --net /tmp/runtime-network.test -test.run '^TestEnsureBridgeNATRemainsBounded$' -test.v + test-boot-regressions: + # Boot regressions from the Aoostar WTR Max outage: the bond rename that + # took it off the network, the DHCP fallback for when the configured + # network doesn't come up, and kernels left without DKMS modules. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: DHCP fallback script + run: sh packaging/tests/network-fallback-test.sh + - name: DKMS modules for kernels configured before their headers + run: sh packaging/tests/dkms-autoinstall-test.sh + - name: NIC naming link files never rename a bond (real udev) + run: sudo bash scripts/ci/test-link-bond-rename.sh + package: # The smoothnas package is otherwise only built by the release pipeline. # Build it, and install it the way the installer does: with apt, from the @@ -197,7 +211,7 @@ jobs: test -x /usr/sbin/tierd test -f /usr/share/tierd-ui/index.html test -x /usr/lib/smoothnas/docker-lxc-daemon - for unit in tierd tierd-host-init smoothnas-runtime smoothnas-tls-init smoothnas-gpu-init; do + for unit in tierd tierd-host-init smoothnas-runtime smoothnas-tls-init smoothnas-gpu-init smoothnas-network-fallback; do systemctl is-enabled "$unit.service" done # DKMS built both storage modules for SmoothKernel. @@ -205,6 +219,27 @@ jobs: dkms status dkms status -k "$kver" | grep -E "^zfs/.*: installed" dkms status -k "$kver" | grep -E "^smoothfs/.*: installed" + # A kernel configured before its headers boots without ZFS: its + # DKMS hook skipped it, and the SmoothKernel headers package never + # reruns the hook. Recreate that state, prove it took (so the + # checks below cannot pass vacuously), and check that both repair + # paths rebuild the module. + unbuilt() { + dkms unbuild -m "$1" -v "$(dkms status -k "$kver" | sed -n "s|^$1/\([^,]*\),.*|\1|p")" -k "$kver" + if dkms status -k "$kver" | grep -E "^$1/.*: installed"; then echo "$1 is still installed for $kver" >&2; exit 1; fi + if [ -n "$(find "/lib/modules/$kver" -name "$1.ko*")" ]; then echo "$1.ko is still in /lib/modules/$kver" >&2; exit 1; fi + } + # Path 1: any later dpkg run (apt.conf.d/53smoothnas-dkms). + unbuilt zfs + apt-get install -y -qq --reinstall hdparm >/dev/null + dkms status -k "$kver" | grep -E "^zfs/.*: installed" + modinfo -k "$kver" zfs >/dev/null + # Path 2: the smoothnas postinst, which an upgrade runs. Not via apt, + # so the Post-Invoke hook cannot be what rebuilt it. + unbuilt smoothfs + DEBIAN_FRONTEND=noninteractive dpkg-reconfigure smoothnas + dkms status -k "$kver" | grep -E "^smoothfs/.*: installed" + modinfo -k "$kver" smoothfs >/dev/null # The package keeps its own apt source; apt reads it cleanly. apt-get update > /tmp/update.log 2>&1 || { cat /tmp/update.log; exit 1; } if grep -E "^(Err|E):" /tmp/update.log; then exit 1; fi @@ -215,6 +250,18 @@ jobs: with: repository: RakuenSoftware/smoothiso path: deps/smoothiso + - name: Installer pins NICs by their permanent MAC + run: | + # A MACAddress= selector also matches a bond carrying that NIC's + # MAC, and udev renames the bond at boot. + tmp=$(mktemp -d) + cp -a deps/smoothiso/. "$tmp/" + for p in iso/smoothiso-patches/*.patch; do patch -d "$tmp" -p1 --batch --silent < "$p"; done + grep -qxF 'PermanentMACAddress=$mac' "$tmp/installer.sh" + if grep -nE '^[[:space:]]*MACAddress=' "$tmp/installer.sh"; then + echo "::error::the installer writes a MACAddress= link selector" + exit 1 + fi - name: Build the ISO env: SMOOTHISO_DIR: ${{ github.workspace }}/deps/smoothiso diff --git a/README.md b/README.md index 1a159fa8..6b21fd7f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ If you know Linux and you want an appliance that still feels like Linux, this is - Create named storage tiers with slot-based array assignment for `NVME`, `SSD`, and `HDD`. - Run ZFS pools, datasets, zvols, and snapshots alongside the mdadm/LVM path. - Publish storage over SMB, NFS, and iSCSI. -- Schedule backups of your storage with live progress, throughput, and cancel control. +- Schedule backups over NFS, SMB, SSH or Google Drive, with live progress, throughput, and cancel control. Google Drive jobs support optional encryption and an independent recovery kit. - Run co-located apps as managed plugins (local LLM inference, runners, media tools) in isolated containers. - Benchmark local and remote targets with live fio-driven telemetry. - Monitor disk health and system alerts, and spin idle disks down to save power. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 4ddc73cb..c12a3988 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -87,11 +87,11 @@ make install-runtime # installs the daemon + smoothnas-runtime.service ``` The runtime wraps [LXC2Docker](https://github.com/games-on-whales/LXC2Docker) -(defaulting to upstream `main`; pin with `LXC2DOCKER_REF`). Build hosts need +(pinned to a tested upstream commit; override with `LXC2DOCKER_REF`). Build hosts need `golang-go`, `pkg-config`, `build-essential`, and `lxc-dev`; runtime hosts need `lxc`, `lxc-templates`, `skopeo`, `umoci`, `rsync`, `nftables`, -`iptables`, `iproute2`, and `uidmap`. SmoothNAS carries no local LXC2Docker -patches — fixes land upstream and are picked up on the next runtime build. +`iptables`, `iproute2`, and `uidmap`. Runtime fixes are pinned upstream; reviewed downstream corrections in +`runtime/patches/` are applied by the same build path used for releases. See [../runtime/README.md](../runtime/README.md) for detail. ### Kernel and OpenZFS @@ -343,3 +343,74 @@ Two specific cleanup tracks are still open and should be treated as engineering The source deep dive calls both out in more detail: - [../src/README.md](../src/README.md) + +## Google Drive backups + +The **Backups** page supports Google Drive alongside NFS, SMB and rsync. The +appliance needs `rclone >= 1.60`; the SmoothNAS package and installer include it. +For a source checkout on an existing machine, install rclone before creating an +encrypted job or starting a Drive run. + +1. Open [Google Cloud Console](https://console.cloud.google.com/), create or + select a project, and enable the **Google Drive API**. +2. Configure the OAuth consent screen as **External** and publish it to + **Production**. Tokens issued while the app is in Testing expire after seven + days. Personal-use apps can proceed through Google's unverified-app warning. +3. Create an OAuth client of type **TVs and Limited Input devices**. Copy its + client ID and client secret. +4. In **Backups → Google accounts → Connect Google account**, enter a label and + the client credentials. Open the Google link and approve the displayed code. + The NAS polls Google; no public callback URL or HTTPS reverse proxy is needed. +5. Add a backup job with method **Google Drive**, choose the account, an absolute + local path and a relative Drive folder such as `smoothnas/storage`. Choose + **Push** to back up or **Pull** to restore. Restores always copy and never + delete local files. Push deletion mode mirrors the selected folder. +6. Enter any exclude patterns, one per line (for example `/video/**` and + `/models/**`). These are rclone exclusion patterns, not `+`/`-` filter rules. +7. Optionally enable encryption. **Prepare recovery kit** produces a private + `smoothnas-recovery.conf`; download it and confirm it is safely stored before + saving. Preparation expires after 30 minutes and is lost on daemon restart. + If it expires, cancel and reopen the form to prepare another kit. +8. Enable a schedule, select weekdays and set a time in the **NAS's local time + zone**. Existing jobs remain manual until scheduling is enabled. + +Encryption covers file contents, filenames and directory names. Encryption +cannot change after the first run. An encrypted job keeps the Google account +and folder recorded in its recovery kit; create a new job for another target. +The kit includes the OAuth credentials, refresh token and encryption keys, so +keep it private and store it somewhere other than the NAS. The API never +exports saved encryption keys. Losing both the NAS and its kit makes encrypted +backups unrecoverable. + +To restore an encrypted job through SmoothNAS, edit the original job and change +its direction to **Pull**, retaining its encryption setting. A newly created +encrypted job generates new keys and cannot decrypt a different job's backup. + +To restore without SmoothNAS, install stock rclone on another machine and run: + +```bash +rclone --config ./smoothnas-recovery.conf copy sncrypt: /path/to/restore +``` + +Keep the original Google Cloud project and OAuth client. SmoothNAS requests +`drive.file`, so it can only see files created through that project. Connecting +through a different project cannot discover the old backup. Disconnecting an +unused account removes its credentials from SmoothNAS and leaves Drive files +intact. Google access can be revoked separately in Google Account settings; +revocation may also affect other clients in the same project. + +Schedules check every 30 seconds. A missed slot in the last 24 hours is caught +up once, and an already running job causes that slot to be skipped. Repeated +DST hours run once; nonexistent times run at the first valid minute afterward. +A daily Google upload cap ends the run with a continuation summary; the next +scheduled run copies remaining changes. Full Drive storage and other errors +remain failures in run history. + +Credentials and encryption keys use the existing SQLite secrets posture. They +are omitted from JSON, command arguments and run errors. Runs use environment +configuration rather than a persistent rclone configuration file. + +Implementation checks include mocked Google device-flow tests and real rclone +local-backend backup, restore and independent recovery-kit tests. A real Google +account connection, encrypted upload and recovery on a second machine still +need operator validation before a release is described as live-validated. diff --git a/docs/compose-plugins.md b/docs/compose-plugins.md index 26a6e05f..d6f0279c 100644 --- a/docs/compose-plugins.md +++ b/docs/compose-plugins.md @@ -93,3 +93,26 @@ services: SmoothNAS-specific concerns compose doesn't: tiered-volume bind resolution + pin, the cross-project host-port guard, and treating `docker compose ps` as the source of truth for state (cached, with a periodic reconcile sweep). + +## Direct web interfaces + +For an application that needs its own HTTPS origin, declare a direct link +instead of `x-smoothnas.ui` (the iframe/proxy target): + +```yaml +x-smoothnas: + webUI: + scheme: https + port: 8007 + path: / +services: + server: + image: example/server:1.0 + ports: ["8007:8007"] +``` + +`webUI.port` must match a fixed published TCP port; `scheme` is `http` or +`https`, and `path` is an absolute path. The Open action uses the hostname of +the current appliance page and opens a new tab. It does not route through +SmoothNAS nginx or inject authentication. The application owns its certificate +and login. See [the PBS plugin guide](pbs-plugin.md) for a complete example. diff --git a/docs/patches/pbs-manifest-fsync.patch b/docs/patches/pbs-manifest-fsync.patch new file mode 100644 index 00000000..1f846408 --- /dev/null +++ b/docs/patches/pbs-manifest-fsync.patch @@ -0,0 +1,11 @@ +--- a/pbs-datastore/src/backup_info.rs ++++ b/pbs-datastore/src/backup_info.rs +@@ -1008,7 +1008,7 @@ + path.push(MANIFEST_BLOB_NAME.as_ref()); + + // atomic replace invalidates flock - no other writes past this point! +- replace_file(&path, raw_data, CreateOptions::new(), false)?; ++ replace_file(&path, raw_data, CreateOptions::new(), true)?; + Ok(()) + } + diff --git a/docs/pbs-plugin.md b/docs/pbs-plugin.md new file mode 100644 index 00000000..91d4d76a --- /dev/null +++ b/docs/pbs-plugin.md @@ -0,0 +1,539 @@ +# Proxmox Backup Server plugin + +The in-tree **pbs** plugin runs Proxmox Backup Server on SmoothNAS so Proxmox +VE can send VM and container backups to the NAS. It is experimental until the +appliance checks below have passed on the storage configuration in use. +Sending SmoothNAS files to an external PBS is a separate integration. + +## Delivery status + +**The current PBS image is blocked for production use.** A later power-loss +repeat left all nine tested snapshot manifests empty. Their archive indexes +and referenced chunks remained intact, but normal verification failed. The +locally rebuilt manifest-sync candidate passed the additional tests below, but +the default plugin image remains the failing baseline. + +PBS on smoothfs requires every fix through `e85ea3a25126`, including interrupted +movement recovery and a real lower-filesystem `syncfs` barrier. Older development +builds can also omit live snapshots during GC and lose chunks. The startup +directory-sync probe does not detect these other defects; load the required +updated module as well as the backend. + +This is a development branch, not a released appliance feature. The live +results below cover the explicitly listed test baseline. Before shipping: + +- Require smoothfs 0.3.5, released after merging + [the tested storage fixes](https://github.com/RakuenSoftware/smoothfs/pull/209). + It contains tested ref `e85ea3a251262e4bea95efa8862d91258e76f880` and adds a + durable cleanup acknowledgement so an object can move repeatedly, + corrects mkdir failure cleanup across supported kernel versions, and fixes + non-root enumeration of snapshots spread over several backing tiers. It now + reconciles interrupted moves with live kernel placement and flushes each + backing filesystem when `syncfs` completes. +- Publish a PBS image with durable manifest replacement and pin its immutable + digest. [The validated candidate patch](patches/pbs-manifest-fsync.patch) targets + PBS 4.2.0. Its locally built image passed the checks below; the plugin still + references the failing upstream baseline, rather than an unavailable test + registry. Publishing and pinning a suitable image remains a release blocker. +- Require the corresponding `smoothfs-dkms` and `smoothfs-samba-vfs` packages + from the archive. `scripts/ci/smoothfs-ref` requires that tested ref; the + minimum-version check now resolves it to release 0.3.5. +- Build the pinned runtime with the retained downstream bridge patch. The pin + includes the [merged credential-logging fixes](https://github.com/games-on-whales/LXC2Docker/pull/160). +- Validate the resulting appliance package on a fresh guest, including loading + its packaged module and repeating backup, restore, movement and reboot checks. + +The Go module pin alone does not update a running appliance's kernel module. +The plugin refuses a datastore whose module lacks directory-sync support. +Keep the plugin experimental pending sustained-load and retention testing. + +## Install and connect + +1. Open **Plugins → Install**, select **pbs**, and choose the storage tiers for + configuration, state, logs and datastore. Supply an initial administrator + password of at least 12 characters without line breaks, materialise, then start. +2. Click **Open**. PBS opens directly at `https://:8007/` in a new + tab. Accept its self-signed certificate after checking its fingerprint, or + configure a trusted certificate in PBS. The appliance address must be + reachable directly from both your browser and the Proxmox VE nodes. +3. Log in as `admin@pbs` (user `admin`, realm **Proxmox Backup authentication + server**). Add a directory datastore, for example `backups`, with backing + path **`/datastore`**. No host disks need to be exposed to PBS. +4. Configure PBS prune, verification and garbage-collection schedules. PBS owns + these jobs; SmoothNAS owns the container lifecycle and storage placement. +5. Create a dedicated PBS user and API token for Proxmox VE. Give both the user + and its privilege-separated token `DatastoreBackup` on + `/datastore/backups` (or the chosen namespace). Use a separate identity with + the necessary read privileges for restore operations. +6. In Proxmox VE, open **Datacenter → Storage → Add → Proxmox Backup Server**. + Enter the NAS hostname, datastore `backups`, token identity + `user@pbs!token-name`, token secret, and PBS certificate fingerprint. Run a + small VM/container backup and restore it to a new guest before scheduling + production backups. + +The image is the unofficial +[ayufan PBS container](https://github.com/ayufan/pve-backup-server-dockerfiles), +pinned to `v4.2.0-1`. This is not an officially supported Proxmox deployment. +Its public default password is never installed: the plugin bootstraps its own +configuration and salted SHA-512 password hash before starting any PBS daemon. +The container's `backup` account uses UID/GID 1000 to match smoothfs ownership, +while keeping PBS's private directory permissions. Startup also checks datastore +directory `fsync` and refuses filesystems that cannot provide it. +The initial password is a SmoothNAS secret; subsequent password changes belong +in PBS. Restart and reinstall do not reset existing credentials. Configuration +from unrelated PBS installations is refused rather than overwritten. + +The initial memory limit is 4 GiB. Increase it for larger stores; upstream +recommends an additional GiB per TiB. Fast local storage is preferred for PBS's +chunk/index workload. The plugin requests no privileged mode, host network, +raw disk devices, or SMART capabilities. SmoothNAS manages the disks. PBS tape, +in-container disk formatting, ZFS administration, and PAM/shell access are +outside this plugin's scope. + +## Data retention and recovery + +All four volumes are tier-backed binds. **Uninstall preserves them.** Compose +teardown removes containers and project files; these binds are not native +manifest volume rows subject to the installer's recursive deletion. + +Paths are `/.plugins/pbs/compose/{config,state,logs,datastore}`. +Record each volume's tier before uninstall. Reinstall **pbs** with the same +assignments to reuse those paths; its administrator credentials, certificates, +datastore configuration and backup contents remain intact. A different tier +selects a different directory and does not migrate the old data. Reinstalling +also requires reentering any SmoothNAS configuration settings because its +database records and initial secret were removed. + +An abrupt power-off can leave an incomplete snapshot. In live testing PBS +listed the interrupted entry with no archive files or completed manifest; +restore failed, and verification reported that entry as a failure while all +completed snapshots verified successfully. Remove an entry only after +confirming it is the known interrupted backup without a completed manifest, +then rerun verification and restore checks. + +Use the required fixed kernel module for full-store recovery. The extended +test found a smoothfs mkdir error-path lock bug: after `ENOSPC`, some subsequent +lookups blocked, PBS retained a backup-group lock, and an application restart +could not finish. Existing snapshots remained readable, but freeing space did +not repair the damaged lock. The isolated test VM needed a forced reset to +load the corrected module. This was a filesystem defect, rather than evidence +that restarting PBS is sufficient recovery on the affected module. + +Keep configuration and datastore together in a recovery plan. A datastore +recovered without its configuration can be attached to a fresh PBS installation +using PBS's **Reuse existing datastore** option. Keep client-side encryption +keys independently; the server cannot recreate them. Avoid simultaneous PBS +instances writing the same datastore. A second PBS on another machine can +provide an independent copy through PBS sync jobs. + +## Appliance validation + +Use a disposable datastore for these checks. Unit tests exercise bootstrap, +catalog metadata, secret storage and lifecycle routing. Live results from a +fresh VM on the `.252` Proxmox test host are recorded below; other storage and +hardware configurations still need validation. + +On the appliance, first check that the plugin starts with `/run` on tmpfs and +that an empty/short initial password prevents startup. Check that the upstream +default password fails and the chosen password authenticates. Restart PBS and +confirm that certificates, credentials and datastore configuration persist. + +Probe the actual datastore mount from the host (substitute its real tier path): + +```sh +python3 scripts/pbs-plugin-smoke.py \ + --filesystem /mnt/media/.plugins/pbs/compose/datastore +``` + +The probe creates and removes its own temporary directory, checking explicit +timestamp updates, locking, rename and file/directory `fsync`. PBS updates chunk +access times explicitly; `noatime` alone does not establish incompatibility. +Both timestamp updates and timestamp preservation during tier movement still +need to pass real backup/GC/restore checks. + +On a Linux client with the official `proxmox-backup-client`, use credentials +with backup, read and prune privileges in the validation datastore. Set +`PBS_REPOSITORY`, `PBS_FINGERPRINT` and `PBS_PASSWORD_FILE` without putting +the secret in command history. The script creates a unique backup group, +uploads two versions, restores and compares hashes/permissions/symlinks, +prunes only that group, then restores again: + +```sh +export PBS_REPOSITORY='validation@pbs!test@nas.example:validation' +export PBS_FINGERPRINT='' +export PBS_PASSWORD_FILE='/secure/path/pbs-token' +python3 scripts/pbs-plugin-smoke.py --workdir /tmp/pbs-validation-01 +``` + +Run **Verify** with previously verified snapshots included, then **Start GC** +in PBS and wait for both tasks to succeed. Rerun: + +```sh +python3 scripts/pbs-plugin-smoke.py --workdir /tmp/pbs-validation-01 --restore-only +``` + +Repeat after smoothfs tier movement and after uninstall/reinstall with the +same tiers. Also test daemon/host restart, concurrent backups, a full datastore, +and loss of a backing tier. GC has a roughly 24-hour grace period: an immediate +GC/restore pass does not prove eventual reclamation. Run it again after that +period and confirm referenced chunks remain readable and unreferenced chunks +are reclaimed. Do not change the appliance clock to force this test. + +References: [PBS installation](https://pbs.proxmox.com/docs/installation.html), +[datastore requirements](https://pbs.proxmox.com/docs/storage.html), +[maintenance and GC](https://pbs.proxmox.com/docs/maintenance.html), +[Proxmox VE integration](https://pbs.proxmox.com/docs/pve-integration.html). + + +## Live validation — 31 October 2026 + +The run used a fresh Debian 13 VM on `pvetest` (`192.168.1.252`, PVE 9.2.6), +Linux `7.2.11+deb13-cloud-amd64`, LXC2Docker, two separate XFS-backed smoothfs +mounts, and the pinned PBS `v4.2.0-1` image. Native PVE backup/restore used the +host's Proxmox Backup Client 4.2.3. Both backing disks share the host's physical +storage; this run tests behavior, not independent disk-failure protection. + +The initial run found two smoothfs blockers: directory `fsync` returned +`EINVAL`, and reuse of an old backing inode after tier movement redirected +unrelated writes into a moved chunk. The latter was reproduced in a small +regression test before fixing cache invalidation/validation and persisting the +destination object identity. The original failing fixture was discarded, and +the backup/restore tests were repeated with fresh data on the fixed module. + +The successful live run used these dependency refs: + +| Component | Tested ref | Required fixes | +|---|---|---| +| smoothfs | `v0.2.28-0.20261031160744-d0edccd6ea2c` | Directory sync, safe inode reuse after movement, persistent destination identity. | +| LXC2Docker | `ff6d6d8f9f6f62ea4b2fe7592e88a10ed20569bb` | Stop logging full create requests and failure configurations containing credentials. | + +The delivery branch now pins the forward-ported smoothfs fixes at +`v0.3.5-0.20261031200503-e85ea3a25126` in both the Go module and protocol gate. +The older successful matrix used the earlier refs recorded in each section; +the continuation below reports the newer failures and fixes. +The extended run below tests the forward-ported code, movement completion, +full-filesystem mkdir error handling, and directory enumeration. The final +packaged appliance still requires validation after its dependency packages are +published. + +Deploy the updated runtime **and rebuild/load the updated smoothfs module** +with the plugin. Reboot an existing appliance to load its rebuilt module; +installing a new backend binary alone does not replace a loaded kernel module. +The startup sync check rejects the older module without directory-sync support. + +Completed checks on the fixed module: + +| Check | Result | +|---|---| +| Authentication | Chosen password accepted; public image default rejected. Current administrator and token secrets absent from runtime journal. | +| File backups | Two versions backed up; restored hashes, modes and symlinks matched before and after pruning. | +| Tier movement | Every referenced chunk of the retained file archive moved to the other tier using the production movement worker. Restore and PBS verification passed after 2,048 unrelated create/write/unlink cycles. | +| Garbage collection | Only disposable test chunks were artificially aged, without changing the system clock. GC removed two unreferenced chunks (2.001 MiB), retained 142 live chunks, and restore/verification passed. | +| Proxmox containers | `vzdump` to PBS and `pct restore` of a fresh 515.6 MiB unprivileged CT passed; sentinel bytes, mode and mapped ownership matched. | +| Proxmox VMs | `vzdump` to PBS and `qmrestore` of a fresh stopped VM with a 1 GiB disk passed; a random disk payload matched byte-for-byte. | +| Uninstall/reinstall | Checked configuration and live chunks retained byte-for-byte; the original password, certificate, token and backup remained usable. A different initial password supplied on reinstall did not reset the existing password. | +| Application restart | Restart with the image's `SIGHUP` stop signal passed. | +| VM reboot | The rebuilt module and two-tier pool mounted, PBS restarted automatically, the same certificate and credentials worked, and restore plus verification of all three backup groups passed. | +| Kernel regressions | Directory sync, actual source-inode reuse, open-file cutover, concurrent getattr/cutover and nested movement passed. | + +Cleanup was verified: all five temporary guests, their disks, temporary PBS +storage entries, credentials, leases, snippets and host scratch files were +removed. The host's existing guests were left running. + +The plugin remains experimental. This initial run did not cover full-datastore +failure, backing-disk loss, abrupt power failure, sustained load, or every lower +filesystem; the extended run below adds several of these checks. The immediate GC test exercises the age cutoff with disposable +chunk timestamps; it does not replace a multi-day retention soak. + +## Extended live validation — 31 October 2026 + +A second fresh guest on `.252` ran the delivery branch with smoothfs based on +0.3.4, Linux `7.2.11+deb13-cloud-amd64`, two 8 GiB XFS backing disks, and the +pinned PBS image/runtime plus the retained runtime bridge patch. All three +virtual disks used ZFS `sync=standard` for the tests. Root-disk sync was disabled +only during initial package provisioning and restored before loading the pool. + +The first 412 referenced chunks moved successfully, but an immediate reverse +move failed with `EBUSY`: the kernel remained `switched` after userspace had +finished cleanup. Commit `e2d838d1fe8f` adds `MOVE_COMPLETE` after source unlink +and directory sync, then returns the object to `placed`. A failed unlink or +acknowledgement remains an incomplete cleanup. New worker tests cover these +failures; the live kernel regression covers four moves with a held descriptor, +premature/stale acknowledgements, and an idempotent completion retry. + +Checks completed on this fixture: + +| Check | Result | +|---|---| +| Concurrent backups | Four clients completed nine rounds: 36 successful file snapshots, including shared-content deduplication and incremental changes. Twelve of these ran after the final directory-enumeration fix. | +| Restore integrity | All 12 initial snapshots restored with matching hashes, modes and symlinks. Four more restores passed during movement; pruning retained exactly the latest two snapshots per group, and all eight retained snapshots restored. | +| Repeated movement | On the fixed module, 32 retained chunks each completed three additional moves while four restores ran. Both kernel and SQLite states returned to `placed` after every move. | +| Encrypted backups | Correct-key restore matched hashes, modes and symlinks; a wrong key was rejected. All three referenced encrypted-archive/catalog chunks moved in both directions, and encrypted restore passed afterwards. | +| Native Proxmox backups | Fresh CT `vzdump`/`pct restore` and VM `vzdump`/`qmrestore` passed, including sentinel bytes/mode/ownership and raw-disk payload checks. A VM-create attempt hit the shared host's storage lock; its serial rerun passed. | +| Cancellation | An active client was interrupted with SIGINT; retrying its 512 MiB archive and restoring it passed. | +| Forced guest power-off | The guest was stopped during a throttled upload. Boot ID changed; the pool mounted and PBS restarted automatically. All eight retained snapshots restored, the 512 MiB upload retried and restored, and the encrypted archive restored. The known incomplete snapshot required explicit removal before verification passed. | +| Normal maintenance | Verification passed during movement. GC respected its age cutoff and queued 464.281 MiB in 116 unreferenced chunks without prematurely deleting recent data. | +| Kernel/runtime checks | Directory sync, inode reuse, open-FD/getattr/nested cutover, repeated movement, and the isolated network-namespace NAT test passed. Go tests, race tests, static checks and kernel builds passed. The 4 GiB container memory limit was enforced; observed final peak was 3.24 GiB, with no OOM events. | + +The capacity test exposed another kernel defect. Linux 6.18 consumes the child +dentry on `vfs_mkdir` failure while leaving its parent locked; 6.19+ also unlocks +the parent. The old materialization path unlocked twice on newer kernels, while +the direct mkdir path failed to unlock on 6.18. Commit `13b3b03983c7` normalizes +the error contract in the compatibility helper. A small two-tier ENOSPC probe +hung on the old module and passed on the fix, including responsive lookup and +write recovery. Builds passed against both 6.18 and 7.2 headers; live regression +execution used 7.2.11. See the primary +[6.18 implementation](https://github.com/torvalds/linux/blob/v6.18/fs/namei.c) and +[6.19 implementation](https://github.com/torvalds/linux/blob/v6.19/fs/namei.c). + +With the mkdir fix, both backing filesystems were filled to 32 MiB free. PBS +correctly failed a 512 MiB upload with `ENOSPC`. After removing only the test +fillers, existing snapshots restored and the same group's new 512 MiB backup +and restore succeeded without restarting PBS. The original failing run had +kept its group locked and blocked application shutdown; a forced VM reset was +needed to discard that damaged kernel state before loading the fix. + +Post-GC checksum restore then exposed a separate data-loss defect. PBS's +UID 1000 could enumerate only the canonical directory below root-owned private +backing parents, while a root probe saw the union of both tiers. A snapshot +created on the other tier after full-store recovery disappeared from PBS's +listing. Verification incorrectly returned `TASK OK` with zero snapshots in +that group, and GC deleted 134 archive chunks and one catalog chunk still +referenced by the snapshot. The failed snapshot's manifest, indexes and failure +logs were preserved as test evidence before its metadata was explicitly removed; its lost chunks +were not recovered. + +Commit `65794ec7eba2` performs backing traversal with the mount credentials, +matching normal lookup, and propagates contributing-tier errors rather than +returning a partial listing. Upper DAC still applies. The old module failed a +non-root merged-directory regression; the corrected module passed for UID 1000, +a bind mount, denial of UID 1001, and propagation of a contributing-tier lookup +error. With the corrected traversal PBS saw the previously hidden snapshot and +verification correctly flagged its missing data. + +Acceptance checks now compare PBS's listing with the exact known completed +snapshot set, require verification to visit every expected snapshot, and run +checksum restores after GC. A successful task status or group count alone is +insufficient. + +The final repeat used every fix through `65794ec7eba2` and rebuilt the lost test +group with a new backup. Full-store failure, freeing space, retry and checksum +restore passed without a restart. Another twelve concurrent file backups and +pruning produced 14 known completed snapshots across ten groups. PBS listed +all 14; verification visited every one. GC then removed 83 unreferenced chunks +(339.585 MiB), retained 819 chunks, and strict verification plus checksum +restores of all eight retained file snapshots, all three recovery archives and +the encrypted archive passed. + +Managed uninstall/reinstall retained seven configuration files and ten selected +live chunks byte-for-byte. The original password, certificate and token remained +usable; a different initial password supplied on reinstall did not reset the +installation, and both that replacement initial password and the image's public +default were rejected. + +A final forced guest power-off interrupted another throttled upload on the final +module. Boot ID changed, the same module and pool loaded, and PBS restarted +automatically. After confirming and removing only the new incomplete snapshot, +the upload retried and restored successfully. PBS listed all 15 expected +completed snapshots across ten groups, and verification visited every snapshot +with zero errors. All eight retained file snapshots, four recovery archives and +the encrypted archive restored with matching data. Native `pct restore` and +`qmrestore` of the original CT/VM backups also passed again after GC, reinstall +and this power-off. Runtime journals contained neither the original nor the +replacement administrator password nor the API token. + +The corrected repeat validates the tested source and fixture. Multi-day retention, +physical backing-tier loss, host power failure and the final packaged appliance +remain unvalidated. Both backing disks share one physical host storage pool. +The package gate still refuses to ship this dependency ref until a smoothfs +release and its archive packages contain all the fixes. + +Cleanup of the extended fixture was verified: VM 9601, both native source/restore +pairs, all owned volumes, temporary PBS storage entries and credentials, leases, +cloud-init snippet, host scratch directory and local private credential copies +were removed. The pre-existing VM 8168 and CT 9474 remained running. Sanitized +failure and success logs were retained locally for review. + + +## Continued failure testing — 31 October 2026 + +A third fresh guest on `.252` used Linux 7.2.11 and two dedicated XFS backing +filesystems. All virtual disks used ZFS `sync=standard` during testing. The +initial root-disk package provisioning and later PBS compilation temporarily +disabled root-disk sync, restoring it before validation. Both backing disks +remained on normal sync throughout. Existing guests were left alone. + +Four clients completed another six rounds of file backups: 24 successful +224 MiB snapshots. Twelve initial restores matched, then another concurrent +backup cycle ran alongside GC. Pruning retained exactly two snapshots per group, +and all eight retained snapshots restored. A separate archive roundtrip preserved +POSIX access/default ACLs, binary user xattrs, UID/GID, setuid/setgid modes, hard +links, Unicode names, dangling symlinks, a FIFO, sparse-file contents and empty +files/directories. + +### Interrupted movement + +Stopping the worker after source cleanup but before its completion acknowledgement +reproduced a live bug: the old startup recovery reported `placed` in SQLite while +the kernel stayed `switched`, so the next movement failed with `EBUSY`. The backup +remained readable. Commit `bbf034674eea` fixes the actual startup service path: + +- Persist a full-sync movement intent and its log before the first kernel plan. +- Use kernel placement to resolve the cutover/database transition gap. +- Remove the non-authoritative copy and sync its parent before acknowledging + completion or rollback; leave failed or uncertain recovery pending. +- Instantiate a persisted path when the post-boot OID index is cold, verify its + identity, then inspect it through the expected pool. + +The original stuck object recovered without rebooting, then moved successfully. +Four worker-exit cases passed: after planning, after copying, after cutover before +its SQLite transition, and immediately before the cleanup acknowledgement. Each +allowed subsequent movements and preserved all 18 retained archive indexes and +267 referenced chunks byte-for-byte. Live guards rejected stale rollback and +rollback after cutover; retries after a completed rollback were idempotent. The +standalone kernel probe passed four rollbacks and four moves with a held file +descriptor. Go tests, race checks, static checks, and builds against Linux 6.18 +and 7.2 headers passed. + +### Corruption detection and power-loss failure + +A deliberate one-byte mutation in a disposable referenced chunk caused verification +to fail and PBS to quarantine that chunk as `.0.bad`. Restore rejected the affected +snapshot. The test validated the quarantined bytes against the exact injected +mutation, restored the saved original, and checked its hash. Restore and strict +verification of all nine expected snapshots then passed; all 18 archive indexes +and 267 referenced chunk hashes still matched. + +The subsequent forced guest power-off exposed a separate critical failure. The +pool/runtime restarted automatically, cold-index movement recovery completed, +and the next movements preserved the referenced data. However, all nine +`index.json.blob` manifests were now zero length, on both the presented filesystem +and its raw XFS backing. Verification failed for every snapshot. The empty +manifests, original archive indexes, hashes and failure logs were preserved as +evidence. The manifests were not reconstructed, and these snapshots were not +claimed as recovered or successfully restored. + +Two findings require separate treatment: + +1. Smoothfs `syncfs` returned success without flushing lower file data. An isolated + XFS test demonstrated delayed allocation remaining on both backing filesystems + and an upper-written file after successful `syncfs`. Commit `e85ea3a25126` now + flushes every distinct lower superblock and reports errors. The old test failed; + the corrected module passed. In a power-off comparison, the old callback lost + all 16 new upper replacements, retaining their older contents. With the fix, + a single upper `syncfs` preserved all 32 new control and upper replacements. +2. PBS 4.2.0 replaces manifests without requesting file sync when updating + verification state. The exact release's + [manifest update](https://git.proxmox.com/?p=proxmox-backup.git;a=blob;f=pbs-datastore/src/backup_info.rs;hb=v4.2.0) + calls `replace_file` with sync disabled; its + [verification path](https://git.proxmox.com/?p=proxmox-backup.git;a=blob;f=src/backup/verify.rs;hb=v4.2.0) + invokes that update. A direct-XFS control reproduced eight empty unsynced + replacements after power loss, while all eight file-synced controls survived. + This supports the manifest-durability diagnosis; the filesystem fix alone is + not acceptance evidence for an image that does not request a barrier after + verification. The candidate patch enables file sync before manifest rename. + +### Backing-tier I/O failure and GC + +A separate valid datastore held one 64 MiB file archive with two indexes and +19 referenced chunks. Its indexes stayed on the slow tier; one live chunk was +moved to the fast tier alongside a disposable orphan. All 20 test chunks were +aged three days while preserving mtime, so an incomplete reference scan could +have deleted the live fast-tier chunk. + +A reversible device-mapper error target made the slow tier return actual `EIO`. +GC failed during phase 1 while opening the dynamic index. Both the live chunk +and the orphan on the healthy fast tier remained present. After restoring the +linear mapping, listing and checksum restore succeeded without restarting PBS. +Both indexes and all 19 chunk hashes matched. Healthy GC then removed exactly +one orphan (1.716 MiB), retained the referenced chunk, and verification plus +another actual restore passed. This separate datastore does not repair or +supersede the nine failed manifests described above. + +### Built manifest-sync candidate + +The candidate was built from the exact PBS `v4.2.0` source commit +`035c449897fafc228c8bbf3a5b5ba38564478ac7`, with only the single-line manifest-sync +patch. All three binaries were rebuilt: API, proxy and manager. Their dynamic +library dependencies resolved inside the clean runtime image, and deployed binary +hashes matched the build outputs. The added image layer contained exactly these +three binaries, with no live container configuration or credentials. + +The base was pinned to +`sha256:01f685a010025233b649417a317cc175b27920cd64f0429abfdc46a0d8d2e88a`. +The tested registry manifest digest was +`sha256:fb5b209705b6b169b374e1b55e15cb39c38eaf5e2b335bfeff5e7cf9aee43edc`. +The image was installed through the actual plugin installer, lifecycle and +Compose backend. Existing TLS identity, token and administrator credentials +survived. A syscall trace observed a successful temporary-manifest `fsync` +before its rename, where the original image's trace had no such sync. + +A fresh candidate datastore passed: + +- Four concurrent file clients, three rounds of 224 MiB archives: all 12 initial + restores matched. Pruning retained exactly two versions per group, and all + eight retained archives restored again. +- Rich metadata and encrypted backup/restore, including wrong-key rejection. +- Native Proxmox CT backup and `pct restore` of a 515.638 MiB OS filesystem, + with matching sentinel content, UID and mode. The backup took 322 seconds + under heavy shared-host I/O pressure; this is correctness evidence, not a + performance measurement. +- Native VM backup and `qmrestore` of a 1 GiB logical disk, with its random + payload matching after restore. +- Exactly 12 retained snapshots across eight groups. The inventory includes + dynamic and fixed indexes for file, CT and VM archives: 23 indexes and 391 + distinct referenced chunks. +- The production movement worker moved 24 referenced chunks and eight retained + archive indexes to the slow tier while the CT backup was active. Tiered file + and encrypted restores passed. +- GC with all 435 test chunks aged three days while preserving mtime. It removed + 44 unreferenced chunks (179.109 MiB), including the control orphan. All 23 + retained indexes and 391 referenced chunk hashes still matched. + +The candidate passed three power-loss tests: an initial one-snapshot smoke test +and two full 12-snapshot repeats after GC. The second full repeat followed +uninstall/reinstall with a different initial password: the original password and +TLS identity survived, and the replacement password and public default were +rejected. Each forced power-off immediately followed +verification, without an added sync barrier. Pool, runtime and PBS restarted +automatically. Strict verification visited every expected snapshot; actual file, +encrypted, rich-metadata, native CT and native VM restores passed after the full +power cut. Each full repeat used fresh native restore guests, then destroyed +them and their temporary PBS storage entry. The 23-index/391-chunk inventory +also matched after both repeats. The nine original empty +manifests were not reconstructed and are not counted as recovered. + +[The amd64 build recipe](../scripts/pbs-image/Dockerfile) records the source, +patch, base and compatible Debian Rust dependency versions. The tested build ran +these commands in a disposable Debian 13 guest and repacked the clean base with +`umoci`; the Dockerfile itself has not been executed. Cargo's lockfile, package +versions, binary hashes and an OCI image archive are retained with the test +evidence. Remaining rolling package inputs mean this is not a claim of a +bit-for-bit reproducible rebuild. + +The final loaded module matched the required build. The final PBS cgroup had a +4 GiB memory limit with no OOM events or kills, and runtime journals contained +neither test administrator password nor the API token. Linux 7.2 emitted one +nonfatal heat-drain workqueue deprecation notice; the final kernel audit found +no BUG, Oops, blocked-task or OOM diagnostic. + +Candidate deployment also reproduced a separate, unfixed runtime issue: +registry addresses containing `:port` are split incorrectly during image pull. +The test used a fixture-local hostname on standard HTTPS after capturing the +failure. This does not establish support for registry-port references. + +Final cleanup was independently audited. It caught one native restore VM missed +by a test-helper ownership flag; that flag was corrected and the verified-owned +VM removed. Main fixture teardown also needed a retry to clear a Proxmox +`lock: destroyed` marker after its disks were removed. The final inventory +confirmed all owned guests, volumes, temporary storage entries and credentials, +leases, mounts, helper process, backup logs, cloud-init snippet and host scratch +were gone. Pre-existing VM 8168 remained running. The host audit separately +records CT 9474's removal by its own `#3088` validation run at 22:23:45; these PBS +tests did not modify that guest. Sanitized evidence and the candidate OCI image +were retained locally. + +Multi-day retention, physical tier loss, host power failure and the packaged +appliance remain unvalidated. Publishing/pinning the patched image and verifying availability of the +required smoothfs 0.3.5 packages are still required. The plugin's installation +description continues to state the observed default-image corruption risk. diff --git a/docs/proposals/done/backup-01-schedule.md b/docs/proposals/done/backup-01-schedule.md new file mode 100644 index 00000000..7cd58cad --- /dev/null +++ b/docs/proposals/done/backup-01-schedule.md @@ -0,0 +1,104 @@ +# Proposal: Backups — Daily Schedules + +**Status:** Implemented, including restart, overlap and DST tests. + + +## Problem + +A backup job only runs when someone presses **Run** in the UI or calls +`POST /api/backup/configs/{id}/run`. Nothing in tierd starts a job on its own. +Several comments (`PurgeBackupsUnderPath`, `destroyTierPool`) already talk about +"the backup schedule", but no scheduler exists. + +An offsite backup that depends on someone remembering to press a button is not +a backup. The Google Drive method (backup-02) needs this most: Google caps +uploads at roughly 750 GB per day, so a first upload of a few terabytes only +finishes if the job runs again every day until it is done. + +## Decisions (locked) + +- **Per-job, all methods.** The schedule is a property of a backup config, not + of a method. `cp`, `rsync`, and the new `rclone` method all get it. +- **Daily at a set time, on chosen weekdays.** No cron expressions. One time of + day plus a weekday set covers the real cases, and is easy to show and validate + in the UI. +- **Catch up once, never stack.** If tierd was down when a job was due, it runs + once when tierd comes back (if the missed slot was in the last 24 hours). Missed + days are never replayed one after another. +- **Never overlap.** If the job already has a run in progress when it comes due, + the slot is skipped, and the next slot is the next chance. + +## Specification + +### Storage + +Migration `00030_backup_schedule.sql` adds four columns to `backup_configs`: + +| Column | Type | Meaning | +|---|---|---| +| `schedule_enabled` | INTEGER NOT NULL DEFAULT 0 | 0 = manual only (every existing job keeps today's behaviour) | +| `schedule_time` | TEXT NOT NULL DEFAULT '' | `HH:MM`, 24-hour, in the NAS's local time zone | +| `schedule_days` | INTEGER NOT NULL DEFAULT 127 | weekday bitmask, bit 0 = Monday … bit 6 = Sunday | +| `last_scheduled_at` | TEXT NOT NULL DEFAULT '' | RFC 3339 UTC time of the last slot the scheduler claimed | + +`db.BackupConfig` gains `ScheduleEnabled`, `ScheduleTime`, `ScheduleDays`, +`LastScheduledAt`, and a computed, read-only `NextRunAt` filled in by the API. + +### Scheduler + +A new `internal/backup/schedule` component, owned by `BackupHandler` and started +with the daemon: + +- It ticks every 30 seconds. On each tick it lists the configs that have + `schedule_enabled = 1`. +- For each one it computes the most recent due slot at or before now. A slot is + due when it is later than `last_scheduled_at` and no more than 24 hours old. +- To claim a slot, it does a conditional `UPDATE … SET last_scheduled_at = ? WHERE + id = ? AND last_scheduled_at = ?` **before** starting the run. A restart, or a + second tick, can then never start the same slot twice. +- It starts the run through the same `startRun` path that a manual run uses, + so drain handling, resume and progress all behave the same. +- If the config already has a `running` run, it claims the slot and skips it, + and logs one line at info level. Skipping is not an error, and it doesn't + create a failed run. +- It does nothing while the handler is draining (`isDraining()`). + +The NAS's local time zone is the reference, so "02:00" means 02:00 local time, +across DST changes too. A slot that doesn't exist on a spring-forward day runs +at the first valid minute after it. A slot that occurs twice on a fall-back day +runs once. + +### API + +`POST/PUT /api/backup/configs` accept `schedule_enabled`, `schedule_time` and +`schedule_days`. Validation rules: + +- `schedule_time` must match `^([01]\d|2[0-3]):[0-5]\d$` when scheduling is + enabled. +- `schedule_days` must be between 1 and 127 when scheduling is enabled. + +`GET` responses include `next_run_at` (empty when scheduling is off). + +### UI + +The backup config form gains a **Schedule** section: an on/off switch, a time +input, and seven weekday toggles that all default to on. The job list shows +**Next run** next to the last run's status. All new strings go through i18n +like the rest of the Backups page (see i18n-02o). + +## Testing + +- Unit tests for slot computation with a fixed clock: normal days, a weekday + mask, the 24-hour catch-up window, both DST transitions, and a slot claimed + twice. +- Handler test: a scheduled slot with a run already in progress is claimed and + skipped, and no new run row is created. +- Restart test: claim a slot, rebuild the scheduler, and confirm the same slot + does not start again. + +## Out of scope + +- Cron expressions, intervals shorter than a day, and schedules that run more + than once a day. +- Notifications on failure. Run history already records failures, and alerting + belongs in a separate proposal. diff --git a/docs/proposals/done/backup-02-google-drive.md b/docs/proposals/done/backup-02-google-drive.md new file mode 100644 index 00000000..444cd3c4 --- /dev/null +++ b/docs/proposals/done/backup-02-google-drive.md @@ -0,0 +1,240 @@ +# Proposal: Backups — Google Drive Method + +**Status:** Implemented. Automated validation passes; real-account Google Drive validation remains an operator release check. + +**Implementation refinements:** Encryption keys and the recovery kit are prepared through `POST /api/backup/google/recovery` before job creation. The acknowledged preparation is consumed on save, expires after 30 minutes, and is never persisted. Saved keys are never re-exported. Encrypted account/folder destinations stay fixed to match the kit. Excludes use `--exclude-from` so patterns cannot become include rules. Full-storage quota errors remain failures. + + +**Depends on:** backup-01-schedule. + +## Problem + +SmoothNAS backups can only reach another machine over NFS, SMB or SSH. There +is no offsite target. A Google One / Google AI Pro plan gives operators +terabytes of Drive storage, which makes a good offsite copy of the data that +matters: documents, music, books, and the operator's own backups. Bulk media +can stay local or go elsewhere. + +## Decisions (locked) + +- **Engine: rclone.** Add a third backup method, `rclone`, with target type + `gdrive`. Other rclone back ends (B2, S3) can be added later as new target + types, without a new method. +- **Sign-in happens in the UI, using Google's device flow.** Google rejects + OAuth redirect URIs that use plain HTTP, a bare IP address, or a domain + outside the public suffix list. A LAN-only NAS can't receive a normal OAuth + redirect. The device flow needs no redirect: SmoothNAS shows a code, and the + operator approves it at google.com/device on any device. +- **Scope: `drive.file`.** The device flow allows only `drive.file` and + `drive.appdata` for Drive (not full `drive`). With `drive.file`, SmoothNAS can + only see files it created itself, which is exactly what a backup tool needs. +- **Encryption is a per-job toggle,** using rclone crypt. +- **The operator brings their own OAuth client.** SmoothNAS doesn't ship a + shared Google client ID. Each operator creates one in their own Google Cloud + project, so quotas and consent are theirs. + +## Specification + +### Operator setup (documented in OPERATIONS.md, linked from the UI) + +1. In Google Cloud Console, create a project, enable the **Google Drive API**, + and create an **OAuth client ID** of type **TVs and Limited Input devices**. +2. Set the OAuth consent screen to **External** and then **publish it to + Production**. A project left in *Testing* status gets refresh tokens that + expire after 7 days, so backups would silently stop authenticating after a + week. The app doesn't need Google verification for personal use. The + operator just clicks through the "unverified app" warning once. +3. In SmoothNAS, go to **Backups → Google accounts → Connect**, paste the + client ID and secret, then approve the code that's shown. + +The UI warns about step 2 in so many words. It is the one setup mistake that +looks fine on day one and breaks a week later. + +### Google accounts + +Migration `00031_backup_google_accounts.sql`: + +```sql +CREATE TABLE backup_google_accounts ( + id INTEGER PRIMARY KEY, + label TEXT NOT NULL UNIQUE, + email TEXT NOT NULL DEFAULT '', + client_id TEXT NOT NULL, + client_secret TEXT NOT NULL, + refresh_token TEXT NOT NULL, + created_at TEXT NOT NULL +); +``` + +Several jobs can share one account. Deleting an account that is still in use +by a job is refused (409), and the response lists the jobs using it. + +Sign-in endpoints, under `/api/backup/google/`: + +| Endpoint | Does | +|---|---| +| `POST device/start` `{label, client_id, client_secret}` | Calls `https://oauth2.googleapis.com/device/code` with scope `openid email https://www.googleapis.com/auth/drive.file`. Returns `{session_id, user_code, verification_url, expires_in}`. | +| `GET device/{session_id}` | Returns `pending`, `connected` (with the account), `denied`, or `expired`. | +| `DELETE device/{session_id}` | Abandons the attempt. | + +tierd polls Google's token endpoint itself, at the `interval` Google returns, +and backs off on `slow_down`. The browser only polls tierd. Pending sessions +live in memory and are lost on restart, so the operator just starts again. When +sign-in succeeds, tierd reads the email from the ID token for display, and +stores only the refresh token. Access tokens are never persisted. + +Implement this with `golang.org/x/oauth2` (`Config.DeviceAuth` / +`DeviceAccessToken`), which is a new direct dependency. + +### Job fields + +`backup_configs` gains: + +| Column | Meaning | +|---|---| +| `google_account_id` | FK to `backup_google_accounts`; required when `target_type = gdrive` | +| `excludes` | newline-separated rclone filter patterns, e.g. `/video/**` | +| `encrypt` | 0/1. **Fixed after the first run.** Changing it would mix plaintext and ciphertext in one destination. | +| `crypt_password`, `crypt_salt` | generated by tierd when `encrypt = 1`; never returned by the API after creation | + +`remote_path` is the folder in Drive (created if missing). `local_path`, +`direction`, `parallelism`, `delete_mode` and the schedule keep their current +meanings. + +### Encryption and the recovery kit + +When a job is created with encryption on, tierd generates the crypt password +and salt (32 random bytes each, base64). The create response, and only the +create response, includes a **recovery kit**. The UI makes the operator download +it, or confirm they've copied it, before the job can be saved. + +The recovery kit is a text file containing: + +- a ready-to-use `rclone.conf` snippet (a crypt remote wrapping a `drive` remote, + with the passwords already obscured in rclone's own format); +- one paragraph explaining how to restore from any machine with rclone. + +**If the NAS is lost and nobody kept the recovery kit, the encrypted backup can't +be recovered.** The UI says so next to the toggle. Filename and directory-name +encryption use rclone's `standard` mode. + +### Running rclone + +Debian packaging: add `rclone` to `Depends` in `tierd/debian/control`. During +implementation, confirm that trixie's packaged rclone supports `scope = +drive.file`, `--drive-stop-on-upload-limit` and `--use-json-log`. If it doesn't, +pin a minimum version. + +tierd never writes an rclone config file. Each run gets its remotes through +environment variables only, which only root can read (`/proc//environ`): + +``` +RCLONE_CONFIG_SNDRIVE_TYPE=drive +RCLONE_CONFIG_SNDRIVE_CLIENT_ID=… RCLONE_CONFIG_SNDRIVE_CLIENT_SECRET=… +RCLONE_CONFIG_SNDRIVE_SCOPE=drive.file +RCLONE_CONFIG_SNDRIVE_TOKEN={"refresh_token":"…","token_type":"Bearer","expiry":"0001-01-01T00:00:00Z"} +# encrypt=1 only: +RCLONE_CONFIG_SNCRYPT_TYPE=crypt +RCLONE_CONFIG_SNCRYPT_REMOTE=sndrive: +RCLONE_CONFIG_SNCRYPT_PASSWORD= RCLONE_CONFIG_SNCRYPT_PASSWORD2= +``` + +rclone refreshes the access token in memory for the run. The refresh token +never changes, so nothing has to be written back. + +Commands: + +| Direction | delete_mode | Command | +|---|---|---| +| push | off | `rclone copy ` | +| push | on | `rclone sync ` (the existing `guardDeleteMode` rules apply) | +| pull (restore) | — | `rclone copy `. Pull never deletes. | + +Common flags: `--transfers --checkers <2×parallelism> +--fast-list --drive-stop-on-upload-limit --use-json-log --stats 5s +--stats-log-level NOTICE`, plus one `--filter-from` file (0600, deleted when the +run ends) built from `excludes`. The process runs under `ioprioIdleCmd`, like +rsync. + +Progress: parse the JSON stats lines (`stats.bytes`, `totalBytes`, +`transfers`, `totalTransfers`, `speed`) into the existing progress callback, so +the run view looks the same as rsync's. + +**Daily upload limit.** When `--drive-stop-on-upload-limit` stops the run, it +ends as **completed** with the summary *"Stopped at Google's daily upload limit +— the next scheduled run continues."* Hitting a quota Google applies on purpose +isn't a failure, and reporting it as one would make every day of a first upload +look broken. + +**Redaction.** Before rclone's stderr goes into a run's `error` or `summary`, the +client secret, refresh token and crypt passwords are removed from it. This +reuses the approach in `plugin/compose.redactSecrets`. + +### Secrets posture + +Client secrets, refresh tokens and crypt passwords are stored as plain SQLite +columns, the same way `smb_pass`, `ssh_pass` and plugin secrets are stored +today. They're never serialized to JSON (`json:"-"`). Sealing secrets at rest +is the hardening follow-up that the plugin-secrets migration already mentions, +and it should cover all of these at once. + +### UI + +- **Backups → Google accounts:** a list with label, email, the number of jobs + using each account, and Disconnect. **Connect** opens a dialog: label, client + ID, client secret, the Production warning, then the device code with a link + to Google's verification URL and a live status. +- **Backup form:** the method picker gains **Google Drive**. Picking it shows: + account, Drive folder, excludes (a textarea with an example), Encrypt (locked + after the first run), delete mode, parallelism and the schedule. +- **Recovery kit step** for encrypted jobs, as described above. +- All new strings go through i18n. + +### Example: the job that motivated this + +| Field | Value | +|---|---| +| local_path | the `storage` share's mount on the reinstalled NAS (path to be set then) | +| remote_path | `smoothnas/storage` | +| excludes | `/video/**`, `/bittorrent/**`, `/proxmoxbackup/**`, `/isos/**`, `/models/**` | +| encrypt | per operator | +| schedule | daily 02:00 | + +That's about 2.2 TiB, which takes roughly 3–4 scheduled runs at the daily +upload limit and then only uploads changes. + +## Testing + +- **Unit:** + - environment construction, with and without crypt; + - the argument matrix (direction × delete_mode × excludes); + - parsing JSON stats into progress; + - mapping the upload-limit stop to *completed*; + - redaction, using every secret kind. +- **Device flow:** an `httptest` stand-in for Google's device/code and token + endpoints covering `authorization_pending`, `slow_down`, `access_denied`, + `expired_token` and success. +- **rclone integration (CI):** run real rclone against a `local` back end in + place of `drive`, with and without crypt. Push, then pull to a fresh + directory, and compare checksums. Push again with nothing changed, and confirm + nothing transfers. +- **Live validation (manual, before merge):** on a test VM, connect a real + account, push a small tree with encryption on, restore it on a different + machine using only the recovery kit and stock rclone, and compare checksums. + +## Known limits + +- **`drive.file` belongs to the OAuth client's project.** If the operator later + connects through a client in a *different* Google Cloud project, SmoothNAS + can't see the files the old client created. They're still in the operator's + Drive (visible in the web UI), and the recovery kit, plus a client from the + original project, still restores them. The account dialog says to reuse the + same client. +- The first upload is limited by Google's ~750 GB/day cap, not by SmoothNAS. + +## Out of scope + +- Other rclone back ends (B2, S3, Backblaze Personal). `target_type` leaves + room for them. +- Versioning or retention inside Drive (rclone `--backup-dir`), and bandwidth + limits. Both are small follow-ups once the method exists. diff --git a/iso/smoothiso-patches/0004-match-nic-links-on-permanent-mac.patch b/iso/smoothiso-patches/0004-match-nic-links-on-permanent-mac.patch new file mode 100644 index 00000000..aa595c65 --- /dev/null +++ b/iso/smoothiso-patches/0004-match-nic-links-on-permanent-mac.patch @@ -0,0 +1,17 @@ +--- a/installer.sh ++++ b/installer.sh +@@ -2683,9 +2683,13 @@ + esac + seen_macs="${seen_macs} ${mac}" + ++ # PermanentMACAddress=, not MACAddress=: SmoothNAS gives its bond the ++ # first member's MAC, and bond slaves take the bond's MAC. A match on ++ # the current MAC catches the bond too, and udev renames bond0 to ++ # net0 at boot, which leaves the NAS with no address. + cat > "$TARGET/etc/systemd/network/00-net${idx}.link" << LINK + [Match] +-MACAddress=$mac ++PermanentMACAddress=$mac + + [Link] + Name=net${idx} diff --git a/packaging/build-deb.sh b/packaging/build-deb.sh index cffbca56..d1774ece 100755 --- a/packaging/build-deb.sh +++ b/packaging/build-deb.sh @@ -31,7 +31,7 @@ samba, smbclient, cifs-utils, nfs-kernel-server, targetcli-fb, python3-rtslib-fb nginx, nftables, iptables, iproute2, openssl, curl, \ lxc, lxc-templates, skopeo, umoci, uidmap, docker-cli, \ btrfs-progs, xfsprogs, thin-provisioning-tools, smartmontools, hdparm, nvme-cli, \ -gdisk, fio, psmisc, rsync, sshpass, iperf3, mokutil, unattended-upgrades" +gdisk, fio, psmisc, rsync, rclone (>= 1.60), sshpass, iperf3, mokutil, unattended-upgrades" RECOMMENDS="bcachefs-tools, bcachefs-kernel-dkms, samba-testsuite" [ -x "$TIERD_BIN" ] || { echo "ERROR: tierd binary not found: ${TIERD_BIN}" >&2; exit 1; } @@ -47,7 +47,8 @@ install -D -m 755 "$TIERD_BIN" "$root/usr/sbin/tierd" install -D -m 755 "$RUNTIME_BIN" "$root/usr/lib/smoothnas/docker-lxc-daemon" mkdir -p "$root/usr/share/tierd-ui" cp -r "$UI_DIR/." "$root/usr/share/tierd-ui/" -chmod 755 "$root/DEBIAN/postinst" "$root/usr/sbin/smoothnas-tls-init" "$root/usr/lib/smoothnas/gpu-init.sh" +chmod 755 "$root/DEBIAN/postinst" "$root/usr/sbin/smoothnas-tls-init" "$root/usr/lib/smoothnas/gpu-init.sh" \ + "$root/usr/lib/smoothnas/network-fallback.sh" "$root/usr/lib/smoothnas/dkms-autoinstall.sh" cat > "$root/DEBIAN/control" </dev/null || true # reenable: an older install's links may point at the removed units above. -for unit in smoothnas-gpu-init smoothnas-tls-init smoothnas-runtime tierd-host-init tierd; do +for unit in smoothnas-gpu-init smoothnas-tls-init smoothnas-network-fallback smoothnas-runtime tierd-host-init tierd; do systemctl reenable "$unit.service" >/dev/null 2>&1 || true done +# Build DKMS modules for kernels that were configured before their headers: +# a NAS upgraded that way boots without ZFS. Later dpkg runs are covered by +# apt.conf.d/53smoothnas-dkms. A failed build must not leave smoothnas +# unconfigured; the script reports which kernel failed. +/usr/lib/smoothnas/dkms-autoinstall.sh || true + if [ -d /run/systemd/system ]; then systemctl daemon-reload # --no-block: on an install's first boot this runs inside diff --git a/packaging/smoothnas/etc/apt/apt.conf.d/53smoothnas-dkms b/packaging/smoothnas/etc/apt/apt.conf.d/53smoothnas-dkms new file mode 100644 index 00000000..63aa8ef0 --- /dev/null +++ b/packaging/smoothnas/etc/apt/apt.conf.d/53smoothnas-dkms @@ -0,0 +1,5 @@ +// Build DKMS modules (OpenZFS, smoothfs) for every installed kernel after each +// dpkg run. SmoothKernel's headers package doesn't rerun the DKMS hook, so a +// kernel configured before its headers would otherwise boot without ZFS. +// See /usr/lib/smoothnas/dkms-autoinstall.sh. +DPkg::Post-Invoke { "if [ -x /usr/lib/smoothnas/dkms-autoinstall.sh ]; then /usr/lib/smoothnas/dkms-autoinstall.sh || true; fi"; }; diff --git a/packaging/smoothnas/usr/lib/smoothnas/dkms-autoinstall.sh b/packaging/smoothnas/usr/lib/smoothnas/dkms-autoinstall.sh new file mode 100755 index 00000000..c493d8fb --- /dev/null +++ b/packaging/smoothnas/usr/lib/smoothnas/dkms-autoinstall.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -u + +# Build every DKMS module (OpenZFS, smoothfs, ...) for every installed kernel +# that has its headers. +# +# A kernel's own postinst runs the DKMS hook, which skips the kernel when its +# headers aren't configured yet. Debian's headers packages rerun the hook when +# they're configured; SmoothKernel's bindeb-pkg headers have no postinst. So +# when apt configures linux-image before linux-headers, nothing ever builds +# the modules for the new kernel. That boots a NAS without ZFS: its pools +# don't import, and every share on them fails. +# +# Runs from the smoothnas postinst and after every dpkg run (apt.conf.d +# 53smoothnas-dkms), so modules get built whatever order packages configure +# in. `dkms autoinstall` skips modules already installed, so a run with +# nothing to build costs a status check per kernel. + +MODULES_DIR="${SMOOTHNAS_MODULES_DIR:-/lib/modules}" + +log() { echo "smoothnas-dkms: $*" >&2; } + +command -v dkms >/dev/null 2>&1 || exit 0 + +status=0 +for dir in "$MODULES_DIR"/*; do + [ -d "$dir" ] || continue + kver="$(basename "$dir")" + # build links to the kernel's headers; it dangles until they're + # installed, and then DKMS can't build for this kernel anyway. + [ -f "$dir/build/Makefile" ] || continue + if ! dkms autoinstall -k "$kver"; then + log "could not build DKMS modules for $kver; see /var/lib/dkms/*/*/build/make.log" + status=1 + fi +done +exit "$status" diff --git a/packaging/smoothnas/usr/lib/smoothnas/network-fallback.sh b/packaging/smoothnas/usr/lib/smoothnas/network-fallback.sh new file mode 100755 index 00000000..449f4f63 --- /dev/null +++ b/packaging/smoothnas/usr/lib/smoothnas/network-fallback.sh @@ -0,0 +1,108 @@ +#!/bin/sh +set -u + +# SmoothNAS boot-time network fallback. +# +# A NAS that boots without a usable address can only be reached from its +# console. If no systemd-networkd managed link becomes routable within +# TIMEOUT seconds of a physical NIC seeing carrier, release every physical +# NIC from its bond and DHCP each one on its own. Fallback config lives in +# /run, so it lasts until the next boot and never replaces the operator's +# configuration in /etc/systemd/network. +# +# The clock starts at carrier, not at boot. After a power cut the switch +# often comes up after the NAS, and a correctly configured box mustn't give +# up its static address because the switch was slow. + +TIMEOUT="${SMOOTHNAS_NETWORK_FALLBACK_TIMEOUT:-90}" +INTERVAL="${SMOOTHNAS_NETWORK_FALLBACK_INTERVAL:-2}" +SYS_CLASS_NET="${SMOOTHNAS_SYS_CLASS_NET:-/sys/class/net}" +RUN_NETWORK_DIR="${SMOOTHNAS_RUN_NETWORK_DIR:-/run/systemd/network}" +STATE_FILE="${SMOOTHNAS_NETWORK_FALLBACK_STATE:-/run/smoothnas/network-fallback}" + +log() { echo "smoothnas-network-fallback: $*" >&2; } + +# Physical Ethernet: backed by a device, ARPHRD_ETHER, not wireless. Bonds, +# VLANs, bridges and veths have no device link. +physical_nics() { + for dev in "$SYS_CLASS_NET"/*; do + [ -e "$dev/device" ] || continue + [ "$(cat "$dev/type" 2>/dev/null)" = 1 ] || continue + [ -e "$dev/wireless" ] && continue + [ -e "$dev/phy80211" ] && continue + basename "$dev" + done +} + +any_carrier() { + for nic in $(physical_nics); do + [ "$(cat "$SYS_CLASS_NET/$nic/carrier" 2>/dev/null)" = 1 ] && return 0 + done + return 1 +} + +# A networkd-managed link (not loopback, not an unmanaged container bridge) +# with a routable address means the configured network came up. +managed_routable() { + networkctl list --no-legend --no-pager 2>/dev/null | + awk '$3 != "loopback" && $4 == "routable" && $5 != "unmanaged" { found = 1 } END { exit !found }' +} + +fall_back() { + nics="$(physical_nics)" + if [ -z "$nics" ]; then + log "no physical Ethernet NICs to fall back to" + return 1 + fi + mkdir -p "$RUN_NETWORK_DIR" "$(dirname "$STATE_FILE")" + for nic in $nics; do + # 00- sorts ahead of every tierd and installer file, so networkd + # matches this file first. + cat > "$RUN_NETWORK_DIR/00-smoothnas-fallback-$nic.network" < "$STATE_FILE" + log "configured network did not come up within ${TIMEOUT}s of carrier; DHCP on each NIC: $(echo $nics)" + return 0 +} + +carrier_seen="" +elapsed=0 +while :; do + if managed_routable; then + exit 0 + fi + if [ -z "$carrier_seen" ] && any_carrier; then + carrier_seen=1 + log "carrier is up; waiting ${TIMEOUT}s for the configured network" + fi + if [ -n "$carrier_seen" ]; then + if [ "$elapsed" -ge "$TIMEOUT" ]; then + fall_back + exit $? + fi + elapsed=$((elapsed + INTERVAL)) + fi + sleep "$INTERVAL" +done diff --git a/packaging/smoothnas/usr/lib/systemd/system/smoothnas-network-fallback.service b/packaging/smoothnas/usr/lib/systemd/system/smoothnas-network-fallback.service new file mode 100644 index 00000000..1e94c1c6 --- /dev/null +++ b/packaging/smoothnas/usr/lib/systemd/system/smoothnas-network-fallback.service @@ -0,0 +1,14 @@ +[Unit] +Description=SmoothNAS DHCP fallback when the configured network does not come up +Documentation=file:///usr/lib/smoothnas/network-fallback.sh +After=systemd-networkd.service +Wants=systemd-networkd.service + +[Service] +# simple: it can wait indefinitely for carrier, and must not hold up boot. +Type=simple +ExecStart=/usr/lib/smoothnas/network-fallback.sh +Restart=no + +[Install] +WantedBy=multi-user.target diff --git a/packaging/tests/dkms-autoinstall-test.sh b/packaging/tests/dkms-autoinstall-test.sh new file mode 100755 index 00000000..51cf6421 --- /dev/null +++ b/packaging/tests/dkms-autoinstall-test.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Tests for usr/lib/smoothnas/dkms-autoinstall.sh against a fake +# /lib/modules and a stub dkms. The real-DKMS end-to-end test is in the CI +# package job. +# +# Usage: packaging/tests/dkms-autoinstall-test.sh +set -eu + +HERE="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="$HERE/../smoothnas/usr/lib/smoothnas/dkms-autoinstall.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +failures=0 + +fail() { + echo "FAIL: $case_name: $*" >&2 + failures=$((failures + 1)) +} + +setup() { + case_name="$1" + T="$WORK/$1" + MODS="$T/lib/modules" + BIN="$T/bin" + mkdir -p "$MODS" "$BIN" "$T/usr/src" + : > "$T/calls" + # dkms fails for any kernel listed in $T/fail-kernels. + cat > "$BIN/dkms" <> "$T/calls" +[ "\$1" = autoinstall ] && [ "\$2" = -k ] && grep -qx "\$3" "$T/fail-kernels" 2>/dev/null && exit 1 +exit 0 +STUB + chmod +x "$BIN/dkms" +} + +# kernel headers|no-headers|dangling +kernel() { + mkdir -p "$MODS/$1/kernel" + case "$2" in + headers) + mkdir -p "$T/usr/src/linux-headers-$1" + echo "# kbuild" > "$T/usr/src/linux-headers-$1/Makefile" + ln -s "$T/usr/src/linux-headers-$1" "$MODS/$1/build" + ;; + dangling) + ln -s "$T/usr/src/linux-headers-$1" "$MODS/$1/build" + ;; + no-headers) ;; + esac +} + +run() { + set +e + PATH="$BIN:$PATH" SMOOTHNAS_MODULES_DIR="$MODS" sh "$SCRIPT" 2> "$T/stderr" + status=$? + set -e +} + +calls() { sort "$T/calls" | tr '\n' ';'; } + +# 1. The WTR Max after its Nov 1 upgrade: 7.2.14-83's image was configured +# before its headers, so DKMS skipped it. 7.2.10-76 was removed (rc): its +# directory remains without headers. +setup wtr-max-upgrade +kernel 7.2.10-76-smoothkernel no-headers +kernel 7.2.10-80-smoothkernel headers +kernel 7.2.14-83-smoothkernel headers +run +[ "$status" = 0 ] || fail "exit $status: $(cat "$T/stderr")" +want="dkms autoinstall -k 7.2.10-80-smoothkernel;dkms autoinstall -k 7.2.14-83-smoothkernel;" +[ "$(calls)" = "$want" ] || fail "calls: got '$(calls)', want '$want'" + +# 2. Kernel image installed, headers not yet: its build link dangles. Skip +# it; the run after its headers are configured builds it. +setup headers-not-yet-installed +kernel 7.2.15-84-smoothkernel dangling +kernel 7.2.14-83-smoothkernel headers +run +[ "$status" = 0 ] || fail "exit $status" +[ "$(calls)" = "dkms autoinstall -k 7.2.14-83-smoothkernel;" ] || fail "calls: '$(calls)'" + +# 3. The next dpkg run, once the headers are in: now it builds. +kernel_headers_arrive() { + mkdir -p "$T/usr/src/linux-headers-$1" + echo "# kbuild" > "$T/usr/src/linux-headers-$1/Makefile" +} +kernel_headers_arrive 7.2.15-84-smoothkernel +: > "$T/calls" +run +[ "$status" = 0 ] || fail "exit $status" +grep -qx "dkms autoinstall -k 7.2.15-84-smoothkernel" "$T/calls" || fail "did not build once headers arrived: '$(calls)'" + +# 4. A failed build is reported with its kernel, the other kernels are still +# built, and the exit status says something failed. +setup one-kernel-fails +kernel 7.2.14-83-smoothkernel headers +kernel 7.2.15-84-smoothkernel headers +echo 7.2.14-83-smoothkernel > "$T/fail-kernels" +run +[ "$status" = 1 ] || fail "exit $status, want 1" +grep -q "7.2.14-83-smoothkernel" "$T/stderr" || fail "failure does not name the kernel: $(cat "$T/stderr")" +grep -qx "dkms autoinstall -k 7.2.15-84-smoothkernel" "$T/calls" || fail "stopped after the failure: '$(calls)'" + +# 5. No dkms (smoothnas removed, conffile left): nothing to do, no error. +setup no-dkms +kernel 7.2.14-83-smoothkernel headers +rm "$BIN/dkms" +set +e +PATH="$BIN:/usr/bin:/bin" SMOOTHNAS_MODULES_DIR="$MODS" sh -c 'command -v dkms >/dev/null && exit 99; exec sh "$0"' "$SCRIPT" 2> "$T/stderr" +status=$? +set -e +if [ "$status" = 99 ]; then + echo "skip: no-dkms (host has a real dkms on PATH)" +else + [ "$status" = 0 ] || fail "exit $status" +fi + +# 6. Empty or missing modules directory. +setup no-kernels +rmdir "$MODS" +run +[ "$status" = 0 ] || fail "exit $status" +[ -z "$(calls)" ] || fail "calls: '$(calls)'" + +# 7. The apt hook runs the script, and tolerates it being absent or failing: +# a dpkg run must never fail because of it. +case_name=apt-hook +HOOK="$HERE/../smoothnas/etc/apt/apt.conf.d/53smoothnas-dkms" +grep -q 'DPkg::Post-Invoke' "$HOOK" || fail "53smoothnas-dkms has no DPkg::Post-Invoke" +cmd="$(sed -n 's/.*DPkg::Post-Invoke { "\(.*\)"; };.*/\1/p' "$HOOK" | sed 's/\\"/"/g')" +[ -n "$cmd" ] || fail "could not read the Post-Invoke command" +echo "$cmd" | grep -q '/usr/lib/smoothnas/dkms-autoinstall.sh' || fail "hook does not run dkms-autoinstall.sh: $cmd" +for target in /nonexistent/script /bin/false; do + probe="$(echo "$cmd" | sed "s|/usr/lib/smoothnas/dkms-autoinstall.sh|$target|g")" + sh -c "$probe" || fail "hook fails the dpkg run when the script is $target" +done +grep -qx /etc/apt/apt.conf.d/53smoothnas-dkms "$HERE/../smoothnas/DEBIAN/conffiles" || fail "53smoothnas-dkms is not a conffile" + +# 8. The postinst runs it and can't fail because of it. +case_name=postinst +grep -qx '/usr/lib/smoothnas/dkms-autoinstall.sh || true' "$HERE/../smoothnas/DEBIAN/postinst" || fail "postinst does not run dkms-autoinstall.sh tolerantly" + +if [ "$failures" -ne 0 ]; then + echo "$failures failure(s)" >&2 + exit 1 +fi +echo "dkms-autoinstall: all cases passed" diff --git a/packaging/tests/network-fallback-test.sh b/packaging/tests/network-fallback-test.sh new file mode 100755 index 00000000..991b92b0 --- /dev/null +++ b/packaging/tests/network-fallback-test.sh @@ -0,0 +1,218 @@ +#!/bin/sh +# Tests for usr/lib/smoothnas/network-fallback.sh against a fake +# /sys/class/net and stub networkctl, ip and sleep. +# +# Usage: packaging/tests/network-fallback-test.sh +set -eu + +HERE="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="$HERE/../smoothnas/usr/lib/smoothnas/network-fallback.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +failures=0 + +fail() { + echo "FAIL: $case_name: $*" >&2 + failures=$((failures + 1)) +} + +# nic [master]: a physical Ethernet NIC. +nic() { + d="$SYS/$1" + mkdir -p "$d/device" + echo 1 > "$d/type" + echo "$2" > "$d/carrier" + if [ $# -ge 3 ]; then ln -s "../$3" "$d/master"; fi +} + +# soft : a software link (bond, veth, bridge) with no device. +soft() { + mkdir -p "$SYS/$1" + echo 1 > "$SYS/$1/type" + echo 0 > "$SYS/$1/carrier" +} + +setup() { + case_name="$1" + T="$WORK/$1" + SYS="$T/sys" + BIN="$T/bin" + mkdir -p "$SYS" "$BIN" "$T/run" + : > "$T/calls" + echo 0 > "$T/ticks" + # networkctl list prints what the case put in $T/links. + printf '%s\n' "IDX LINK TYPE OPERATIONAL SETUP" > "$T/links.header" + : > "$T/links" + cat > "$BIN/networkctl" <> "$T/calls" +[ "\$1" = list ] && cat "$T/links" +exit 0 +EOF + cat > "$BIN/ip" <> "$T/calls" +exit 0 +EOF + # Each sleep is one tick; an on-tick hook lets a case change the world. + cat > "$BIN/sleep" < "$T/ticks" +[ -x "$T/on-tick" ] && "$T/on-tick" "\$n" +[ "\$n" -ge 500 ] && { echo "sleep stub: too many ticks" >&2; kill -TERM \$PPID; } +exit 0 +EOF + chmod +x "$BIN/networkctl" "$BIN/ip" "$BIN/sleep" +} + +run() { + set +e + PATH="$BIN:$PATH" \ + SMOOTHNAS_SYS_CLASS_NET="$SYS" \ + SMOOTHNAS_RUN_NETWORK_DIR="$T/run" \ + SMOOTHNAS_NETWORK_FALLBACK_STATE="$T/state" \ + SMOOTHNAS_NETWORK_FALLBACK_TIMEOUT="${TIMEOUT:-10}" \ + SMOOTHNAS_NETWORK_FALLBACK_INTERVAL=1 \ + sh "$SCRIPT" 2> "$T/stderr" + status=$? + set -e +} + +fallback_files() { ls "$T/run" 2>/dev/null | sort | tr '\n' ' '; } +ticks() { cat "$T/ticks"; } + +# 1. The configured network is up: exit at once and touch nothing. +setup routable-at-boot +nic net0 1 bond0 +soft bond0 +echo "9 bond0 bond routable configured" > "$T/links" +run +[ "$status" = 0 ] || fail "exit $status" +[ -z "$(fallback_files)" ] || fail "wrote fallback files: $(fallback_files)" +[ ! -e "$T/state" ] || fail "wrote state file" +grep -q '^ip ' "$T/calls" && fail "changed links: $(cat "$T/calls")" + +# 2. The WTR Max outage: udev renamed bond0 to net0, the real NIC kept +# eth1, nothing managed is routable, and lxcbr0 (unmanaged) has an address. +setup wtr-max-renamed-bond +soft net0 # the renamed bond: no device +nic eth1 0 bond0 # the NIC that should have been net0 +nic net1 1 bond0 # X710 port with link +nic net2 0 net0 # enslaved to the renamed bond +nic net3 0 net0 +soft bond0 # recreated by hand, no address +soft lxcbr0 +soft veth0 +cat > "$T/links" </dev/null)" +[ "$(ticks)" -ge 10 ] || fail "fell back after $(ticks)s, before the 10s timeout" +[ "$(ticks)" -le 11 ] || fail "fell back after $(ticks)s, well past the 10s timeout" + +# 3. No carrier: the switch is still booting. Never fall back, however long. +setup no-carrier +nic net0 0 bond0 +soft bond0 +echo "9 bond0 bond no-carrier configuring" > "$T/links" +cat > "$T/on-tick" < "$T/links" +exit 0 +EOF +chmod +x "$T/on-tick" +TIMEOUT=10 run +[ "$status" = 0 ] || fail "exit $status" +[ -z "$(fallback_files)" ] || fail "fell back without carrier: $(fallback_files)" + +# 4. The clock starts at carrier: the switch comes up at 200s, the bond's +# static address applies 5s later. No fallback. +setup late-switch +nic net0 0 bond0 +soft bond0 +echo "9 bond0 bond no-carrier configuring" > "$T/links" +cat > "$T/on-tick" < "$SYS/net0/carrier" +[ "\$1" -eq 205 ] && echo "9 bond0 bond routable configured" > "$T/links" +exit 0 +EOF +chmod +x "$T/on-tick" +TIMEOUT=10 run +[ "$status" = 0 ] || fail "exit $status" +[ -z "$(fallback_files)" ] || fail "fell back although the network came up within the timeout of carrier: $(fallback_files)" + +# 5. Carrier appears late and the network never comes up: fall back TIMEOUT +# seconds after carrier, not after boot. +setup late-carrier-broken +nic net0 0 bond0 +soft bond0 +echo "9 bond0 bond no-carrier configuring" > "$T/links" +cat > "$T/on-tick" < "$SYS/net0/carrier" +exit 0 +EOF +chmod +x "$T/on-tick" +TIMEOUT=10 run +[ "$status" = 0 ] || fail "exit $status" +[ "$(fallback_files)" = "00-smoothnas-fallback-net0.network " ] || fail "fallback files: '$(fallback_files)'" +[ "$(ticks)" -ge 110 ] && [ "$(ticks)" -le 111 ] || fail "fell back at tick $(ticks), want 10s after carrier at tick 100" + +# 6. A routable static address with an unreachable gateway still counts as up. +setup static-no-gateway +nic net0 1 +echo "2 net0 ether routable configured" > "$T/links" +run +[ "$status" = 0 ] || fail "exit $status" +[ -z "$(fallback_files)" ] || fail "fell back on a routable static address" + +# 7. Only unmanaged links are routable (container bridges): fall back. +setup only-unmanaged-routable +nic net0 1 +soft lxcbr0 +cat > "$T/links" < "$SYS/wlan0/type"; echo 1 > "$SYS/wlan0/carrier" +mkdir -p "$SYS/ib0/device"; echo 32 > "$SYS/ib0/type"; echo 1 > "$SYS/ib0/carrier" +echo "2 net0 ether degraded configuring" > "$T/links" +TIMEOUT=2 run +[ "$(fallback_files)" = "00-smoothnas-fallback-net0.network " ] || fail "fallback files: '$(fallback_files)'" + +if [ "$failures" -ne 0 ]; then + echo "$failures failure(s)" >&2 + exit 1 +fi +echo "network-fallback: all cases passed" diff --git a/runtime/README.md b/runtime/README.md index 25afc3ac..f2764a60 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -19,19 +19,19 @@ The build target writes `bin/docker-lxc-daemon`. Appliance installs place it at `/usr/lib/smoothnas/docker-lxc-daemon` and install `runtime/smoothnas-runtime.service`. -The upstream source is pinned to the reviewed LXC2Docker commit: +The upstream source is pinned to the tested LXC2Docker commit that removes +container request and failure-config logging of credentials: ```text https://github.com/games-on-whales/LXC2Docker.git -7171568494541a1c9a776670a1623f3db64d7e46 +ff6d6d8f9f6f62ea4b2fe7592e88a10ed20569bb ``` Set `LXC2DOCKER_REF=` to evaluate a newer revision. Downstream corrections are applied by the same build path used for releases. - SmoothNAS carries runtime corrections in `runtime/patches/`. The build script applies them after checking out LXC2Docker; a patch that no longer applies fails -the build. The bridge correction replaces only the owned postrouting chain in -one nft transaction, preserving published port DNAT rules and removing old -duplicate masquerade rules. Bridge and port-forward mutations share `nftMu`. +the build. The pinned upstream already bounds masquerade rules in one nft transaction. +The bridge correction serializes bridge and port-forward mutations with +`nftMu` and checks that published port DNAT rules survive repeated reconciliation. diff --git a/runtime/patches/0001-idempotent-bridge-masquerade.patch b/runtime/patches/0001-idempotent-bridge-masquerade.patch index 2bd3d74d..584c740b 100644 --- a/runtime/patches/0001-idempotent-bridge-masquerade.patch +++ b/runtime/patches/0001-idempotent-bridge-masquerade.patch @@ -9,34 +9,6 @@ iface, err := net.InterfaceByName(BridgeName) if err != nil || iface == nil { // Bridge doesn't exist — create it and assign the gateway IP. -@@ -69,18 +71,16 @@ - } - } - -- // Ensure the nftables table exists with the masquerade rule. -- // Using "nft -f" with a table block is idempotent — it merges into any -- // existing table rather than replacing it. -+ // Reconcile only our postrouting chain in one atomic transaction. -+ // A table block appends rules on every call; flush removes old duplicates -+ // while retaining prerouting/output DNAT rules used by published ports. - nftRule := fmt.Sprintf(` --table ip %s { -- chain postrouting { -- type nat hook postrouting priority srcnat; policy accept; -- ip saddr 10.100.0.0/24 oifname != "%s" masquerade -- ct status dnat masquerade -- } --} --`, NATTableName, BridgeName) -+add table ip %s -+add chain ip %s postrouting { type nat hook postrouting priority srcnat; policy accept; } -+flush chain ip %s postrouting -+add rule ip %s postrouting ip saddr 10.100.0.0/24 oifname != "%s" masquerade -+add rule ip %s postrouting ct status dnat masquerade -+`, NATTableName, NATTableName, NATTableName, NATTableName, BridgeName, NATTableName) - cmd := exec.Command("nft", "-f", "-") - cmd.Stdin = strings.NewReader(nftRule) - if out, err := cmd.CombinedOutput(); err != nil { --- /dev/null +++ b/internal/lxc/network_nat_integration_test.go @@ -0,0 +1,51 @@ diff --git a/scripts/build-smoothnas-runtime.sh b/scripts/build-smoothnas-runtime.sh index 427462b7..f9adf4d0 100755 --- a/scripts/build-smoothnas-runtime.sh +++ b/scripts/build-smoothnas-runtime.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# Build the SmoothNAS plugin runtime daemon from a reviewed upstream commit. +# Build the SmoothNAS plugin runtime daemon from a tested upstream LXC2Docker commit. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" LXC2DOCKER_REPO="${LXC2DOCKER_REPO:-https://github.com/games-on-whales/LXC2Docker.git}" -LXC2DOCKER_REF="${LXC2DOCKER_REF:-7171568494541a1c9a776670a1623f3db64d7e46}" +LXC2DOCKER_REF="${LXC2DOCKER_REF:-ff6d6d8f9f6f62ea4b2fe7592e88a10ed20569bb}" BUILD_DIR="${PROJECT_DIR}/runtime/build/LXC2Docker" OUT="${PROJECT_DIR}/bin/docker-lxc-daemon" diff --git a/scripts/ci/smoothfs-ref b/scripts/ci/smoothfs-ref index 82dfc31b..ef9c52ee 100644 --- a/scripts/ci/smoothfs-ref +++ b/scripts/ci/smoothfs-ref @@ -1 +1 @@ -13b4491ba43bca6f6b67f35add7026083505d228 +e85ea3a251262e4bea95efa8862d91258e76f880 diff --git a/scripts/ci/test-link-bond-rename.sh b/scripts/ci/test-link-bond-rename.sh new file mode 100755 index 00000000..c826aeba --- /dev/null +++ b/scripts/ci/test-link-bond-rename.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Real-udev regression test for the bond rename that took the Aoostar WTR Max +# off the network: a NIC naming .link file that matches MACAddress= also +# matches a bond carrying that NIC's MAC, and udev renames the bond. +# +# Runs as root on a host with systemd-udevd and the bonding module (the CI +# runner, or a SmoothNAS box). It only touches a test bond and one +# uniquely named .link file, and removes both on exit. +# +# 1. Negative control: with MACAddress=, udev renames the bond. If this +# does not reproduce, the host can't show the bug and the test fails +# rather than passing vacuously. +# 2. The fix: with PermanentMACAddress=, the bond keeps its name. +# 3. PermanentMACAddress= still matches a real NIC (dry run through +# `udevadm test-builtin`, with Name= set to the NIC's current name, so +# nothing is renamed even if it were applied). +# 4. The link file tierd's migration produces and the one the installer +# writes are what step 2 tested. +set -euo pipefail + +[ "$(id -u)" = 0 ] || { echo "must run as root" >&2; exit 2; } + +LINK_DIR=/etc/systemd/network +LINK="$LINK_DIR/00-smoothnas-ci-bond-rename.link" +BOND=smcibond0 +RENAMED=smcinic0 +# Locally administered, so it can't collide with real hardware. +MAC=02:5a:4e:c1:00:01 + +cleanup() { + ip link del "$BOND" 2>/dev/null || true + ip link del "$RENAMED" 2>/dev/null || true + rm -f "$LINK" + udevadm control --reload 2>/dev/null || true +} +trap cleanup EXIT +cleanup +modprobe bonding +mkdir -p "$LINK_DIR" + +write_link() { # + printf '[Match]\n%s=%s\n\n[Link]\nName=%s\n' "$1" "$2" "$3" > "$LINK" + udevadm control --reload +} + +# Create the bond the way networkd does: the MAC is set in the same netlink +# request, so it's already the member's MAC when udev handles the add event. +bond_name_after_add() { + ip link del "$BOND" 2>/dev/null || true + ip link del "$RENAMED" 2>/dev/null || true + ip link add "$BOND" address "$MAC" type bond mode active-backup + udevadm settle --timeout=30 + if [ -e "/sys/class/net/$RENAMED" ]; then + echo "$RENAMED" + elif [ -e "/sys/class/net/$BOND" ]; then + echo "$BOND" + else + echo "missing" + fi +} + +failures=0 +fail() { echo "FAIL: $*" >&2; failures=$((failures + 1)); } + +echo "== 1. negative control: MACAddress= renames a bond carrying that MAC" +write_link MACAddress "$MAC" "$RENAMED" +got="$(bond_name_after_add)" +if [ "$got" != "$RENAMED" ]; then + fail "with MACAddress= the bond is '$got', want '$RENAMED': this host does not reproduce the bug, so step 2 would prove nothing" +fi + +echo "== 2. PermanentMACAddress= does not match the bond" +write_link PermanentMACAddress "$MAC" "$RENAMED" +got="$(bond_name_after_add)" +[ "$got" = "$BOND" ] || fail "with PermanentMACAddress= the bond is '$got', want '$BOND'" + +echo "== 3. PermanentMACAddress= matches a real NIC" +nic="" +perm="" +for dev in /sys/class/net/*; do + [ -e "$dev/device" ] || continue + [ "$(cat "$dev/type")" = 1 ] || continue + n="$(basename "$dev")" + p="$(ip -d link show dev "$n" | sed -n 's/.*permaddr \([0-9a-f:]*\).*/\1/p')" + # Unenslaved NICs don't print permaddr; their address is the permanent + # one when the kernel says it was assigned by hardware (0 = PERM). + if [ -z "$p" ] && [ "$(cat "$dev/addr_assign_type" 2>/dev/null)" = 0 ]; then + p="$(cat "$dev/address")" + fi + if [ -n "$p" ] && [ "$p" != 00:00:00:00:00:00 ]; then + nic="$n"; perm="$p"; break + fi +done +if [ -z "$nic" ]; then + fail "no physical NIC with a permanent MAC on this host" +else + write_link PermanentMACAddress "$perm" "$nic" + out="$(udevadm test-builtin net_setup_link "/sys/class/net/$nic" 2>&1 || true)" + if ! grep -q "$LINK" <<<"$out"; then + echo "$out" >&2 + fail "PermanentMACAddress=$perm did not match $nic" + fi +fi + +echo "== 4. the shipped link files use the selector step 2 tested" +here="$(cd "$(dirname "$0")" && pwd)" +if ! grep -q 'PermanentMACAddress=%s' "$here/../../tierd/internal/network/linkfile.go"; then + fail "tierd's GenerateLinkFile no longer writes PermanentMACAddress=" +fi +if ! grep -qx '+PermanentMACAddress=$mac' "$here/../../iso/smoothiso-patches/0004-match-nic-links-on-permanent-mac.patch"; then + fail "the installer patch no longer writes PermanentMACAddress=" +fi + +if [ "$failures" -ne 0 ]; then + echo "$failures failure(s)" >&2 + exit 1 +fi +echo "link-bond-rename: all checks passed" diff --git a/scripts/pbs-image/Dockerfile b/scripts/pbs-image/Dockerfile new file mode 100644 index 00000000..bb6db248 --- /dev/null +++ b/scripts/pbs-image/Dockerfile @@ -0,0 +1,47 @@ +# Experimental manifest-durability candidate; build from the repository root: +# docker build -f scripts/pbs-image/Dockerfile -t pbs:4.2.0-manifest-fsync . +# The live validation used these commands in a disposable Debian 13 guest, +# then inserted the resulting binaries into this same clean runtime image. +FROM debian:13 AS build +ARG DEBIAN_FRONTEND=noninteractive +ARG BUILD_JOBS=1 +RUN test "$(dpkg --print-architecture)" = amd64 \ + && apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git dpkg-dev \ + && curl -fsSL https://enterprise.proxmox.com/debian/proxmox-release-trixie.gpg \ + -o /usr/share/keyrings/proxmox.gpg \ + && printf '%s\n' \ + 'deb [signed-by=/usr/share/keyrings/proxmox.gpg] http://download.proxmox.com/debian/pbs trixie pbs-no-subscription' \ + 'deb [signed-by=/usr/share/keyrings/proxmox.gpg] http://download.proxmox.com/debian/devel trixie main' \ + > /etc/apt/sources.list.d/proxmox.list +# PBS 4.2.0 needs the older apt API; newer devel packages use incompatible majors. +RUN printf '%s\n' \ + 'Package: librust-proxmox-apt-dev librust-proxmox-apt+*' 'Pin: version 0.99.7-1' 'Pin-Priority: 1001' '' \ + 'Package: librust-proxmox-apt-api-types*' 'Pin: version 2.0.5-1' 'Pin-Priority: 1001' '' \ + 'Package: librust-pbs-api-types*' 'Pin: version 1.0.15-1' 'Pin-Priority: 1001' '' \ + 'Package: librust-proxmox-upgrade-checks*' 'Pin: version 1.0.1-1' 'Pin-Priority: 1001' '' \ + 'Package: librust-proxmox-openid*' 'Pin: version 1.0.2-1' 'Pin-Priority: 1001' '' \ + 'Package: librust-proxmox-subscription*' 'Pin: version 1.0.1-1' 'Pin-Priority: 1001' \ + > /etc/apt/preferences.d/pbs-build \ + && git clone --branch v4.2.0 --single-branch https://git.proxmox.com/git/proxmox-backup.git /build +WORKDIR /build +RUN test "$(git rev-parse HEAD)" = 035c449897fafc228c8bbf3a5b5ba38564478ac7 \ + && apt-get update \ + && apt-get --solver 3.0 -Pnodoc --no-install-recommends -y build-dep . +COPY docs/patches/pbs-manifest-fsync.patch /manifest-fsync.patch +RUN git apply --check /manifest-fsync.patch && git apply /manifest-fsync.patch \ + && export DEB_VERSION="$(dpkg-parsechangelog -SVersion)" \ + && export DEB_VERSION_UPSTREAM="${DEB_VERSION%-*}" \ + && export REPOID=035c449897fafc228c8bbf3a5b5ba38564478ac7-manifest-fsync \ + && CARGO_BUILD_JOBS="$BUILD_JOBS" CARGO_PROFILE_RELEASE_DEBUG=0 \ + cargo build --offline --release --package proxmox-backup \ + --bin proxmox-backup-api --bin proxmox-backup-proxy --bin proxmox-backup-manager \ + && mkdir /out \ + && cp target/release/proxmox-backup-api target/release/proxmox-backup-proxy \ + target/release/proxmox-backup-manager /out/ +FROM ayufan/proxmox-backup-server@sha256:01f685a010025233b649417a317cc175b27920cd64f0429abfdc46a0d8d2e88a +COPY --from=build /out/proxmox-backup-api /out/proxmox-backup-proxy /usr/lib/x86_64-linux-gnu/proxmox-backup/ +COPY --from=build /out/proxmox-backup-manager /usr/sbin/proxmox-backup-manager +LABEL pbs.candidate="manifest-fsync" \ + pbs.source="035c449897fafc228c8bbf3a5b5ba38564478ac7" \ + pbs.base="sha256:01f685a010025233b649417a317cc175b27920cd64f0429abfdc46a0d8d2e88a" diff --git a/scripts/pbs-plugin-smoke.py b/scripts/pbs-plugin-smoke.py new file mode 100644 index 00000000..2d42e997 --- /dev/null +++ b/scripts/pbs-plugin-smoke.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Validate a PBS plugin datastore and exercise a disposable backup group. + +Credentials use the normal PBS_* environment variables. Run --restore-only +again after PBS verification/GC, tier movement, or plugin reinstall. +""" + +import argparse +import datetime +import fcntl +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + + +def filesystem_probe(path): + # Use a new sibling directory, never modify actual PBS chunks or indexes. + with tempfile.TemporaryDirectory(prefix=".smoothnas-pbs-probe-", dir=path) as tmp: + root = Path(tmp) + source = root / "chunk.tmp" + with source.open("wb") as handle: + handle.write(b"PBS filesystem probe\n") + handle.flush() + os.fsync(handle.fileno()) + chunk = root / "chunk" + source.rename(chunk) + old = time.time_ns() - 3 * 24 * 3600 * 10**9 + os.utime(chunk, ns=(old, old)) + current = time.time_ns() + os.utime(chunk, ns=(current, old)) + st = chunk.stat() + if abs(st.st_atime_ns - current) > 10**9 or abs(st.st_mtime_ns - old) > 10**9: + raise RuntimeError("Explicit chunk atime update did not preserve mtime") + with chunk.open("rb") as first, chunk.open("rb") as second: + fcntl.flock(first, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + fcntl.flock(second, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + pass + else: + raise RuntimeError("Conflicting datastore locks were both granted") + fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + print("Filesystem probe passed: timestamp updates, locking, rename and fsync") + + +def client(args, *command): + cmd = [args.client, *command] + if args.namespace: + cmd += ["--ns", args.namespace] + subprocess.run(cmd, check=True) + + +def contents(root): + result = {} + for path in sorted(root.rglob("*")): + name = str(path.relative_to(root)) + if path.is_symlink(): + result[name] = {"link": os.readlink(path)} + elif path.is_file(): + result[name] = { + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "mode": path.stat().st_mode & 0o777, + } + return result + + +def restore(args, record): + # Keep the source/evidence between invocations; restored trees are disposable. + with tempfile.TemporaryDirectory(prefix="restore-", dir=args.workdir) as tmp: + target = Path(tmp) / "files" + client(args, "restore", record["snapshot"], "data.pxar", str(target)) + if contents(target) != record["contents"]: + raise RuntimeError("Restored hashes, symlinks or permissions differ") + print("Restore passed: hashes, symlinks and permissions match") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--filesystem", type=Path, help="Probe a mounted datastore path locally") + parser.add_argument("--workdir", type=Path, help="New evidence directory for network backup/restore") + parser.add_argument("--restore-only", action="store_true", help="Restore an existing workdir after maintenance/reinstall") + parser.add_argument("--client", default="proxmox-backup-client") + parser.add_argument("--namespace", default="") + args = parser.parse_args() + if not args.filesystem and not args.workdir: + parser.error("provide --filesystem and/or --workdir") + if args.filesystem: + filesystem_probe(args.filesystem) + if not args.workdir: + return + if not os.environ.get("PBS_REPOSITORY"): + parser.error("set PBS_REPOSITORY and PBS credentials in the environment") + evidence = args.workdir / "validation.json" + if args.restore_only: + record = json.loads(evidence.read_text()) + if record["repository"] != os.environ["PBS_REPOSITORY"] or record["namespace"] != args.namespace: + parser.error("repository and namespace must match the saved evidence") + restore(args, record) + return + args.workdir.mkdir(mode=0o700, parents=True, exist_ok=False) + source = args.workdir / "source" + source.mkdir() + (source / "payload").write_bytes(os.urandom(2 * 1024 * 1024)) + (source / "note").write_text("first version\n") + (source / "note").chmod(0o640) + (source / "link").symlink_to("note") + group = "host/smoothnas-validation-" + uuid.uuid4().hex + snapshots = [] + for version in (1, 2): + stamp = int(time.time()) + (source / "note").write_text(f"version {version}\n") + client(args, "backup", f"data.pxar:{source.resolve()}", "--backup-id", group.split("/")[1], + "--backup-type", "host", "--backup-time", str(stamp), "--crypt-mode", "none") + iso = datetime.datetime.fromtimestamp(stamp, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + snapshots.append(f"{group}/{iso}") + if version == 1: + time.sleep(1.1) + record = {"snapshot": snapshots[-1], "contents": contents(source), + "repository": os.environ["PBS_REPOSITORY"], "namespace": args.namespace} + evidence.write_text(json.dumps(record, indent=2) + "\n") + restore(args, record) + # Retention applies ONLY to this new UUID group, never existing backups. + client(args, "prune", group, "--keep-last", "1", "--dry-run", "false") + restore(args, record) + print(f"Backup/prune/restore passed for {group}") + print("Run PBS verification and garbage collection, then rerun with --restore-only.") + print("Repeat the restore after tier movement and uninstall/reinstall on the same tiers.") + + +if __name__ == "__main__": + main() diff --git a/tests/browser/updates_page.py b/tests/browser/updates_page.py index c6af6f01..45c77ed8 100644 --- a/tests/browser/updates_page.py +++ b/tests/browser/updates_page.py @@ -6,8 +6,10 @@ The page lists available updates, checks for and installs them on request while following the run's log, shows a failed run's errors (an unreachable -archive must not be silent), says whether a restart is needed and why, and -restarts only after the operator confirms. +archive must not be silent), says when a failed install has since been +superseded by successful unattended upgrades and when an install succeeded +although a package source couldn't be refreshed, says whether a restart is +needed and why, and restarts only after the operator confirms. """ import json import os @@ -18,6 +20,7 @@ base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:5173" REASON = "smoothfs 0.3.2 is installed; running 0.3.1" RESOLVE_ERR = "Err:5 https://updates.rakuensoftware.com/public common InRelease: Temporary failure resolving 'updates.rakuensoftware.com'" +BCACHEFS_ERR = "Err:4 https://apt.bcachefs.org/trixie bcachefs-tools-release InRelease: Could not connect to apt.bcachefs.org:443 (5.161.236.196). - connect (111: Connection refused)" LOG = "== smoothnas-update start install 2026-10-29T18:10:00Z\nSetting up smoothfs-dkms (0.3.2-1) ...\n" @@ -34,6 +37,23 @@ def status(scenario, state): "last_run": {"action": "install", "started_at": "t0", "finished_at": "t1", "exit_code": 0, "errors": []}} return {"running": False, "available": [], "reboot": no_reboot, "last_run": {"action": "check", "started_at": "t0", "finished_at": "2026-10-29T18:00:00Z", "exit_code": 0, "errors": []}} + # The WTR Max on 2026-11-02: an install started here failed on Nov 1 + # because apt.bcachefs.org was down; unattended upgrades have succeeded + # since. That failure is history, not an error. + if scenario == "install_failed_superseded": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "2026-11-01T15:36:21Z", "finished_at": "2026-11-01T15:36:28Z", + "exit_code": 100, "errors": [BCACHEFS_ERR], "superseded_at": "2026-11-02T13:38:27Z"}} + # Negative control: the same failure, nothing has succeeded since. + if scenario == "install_failed": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "2026-11-01T15:36:21Z", "finished_at": "2026-11-01T15:36:28Z", + "exit_code": 100, "errors": [BCACHEFS_ERR]}} + # A source was down, but the install went ahead and succeeded. + if scenario == "installed_refresh_failed": + return {"running": False, "available": [], "reboot": no_reboot, + "last_run": {"action": "install", "started_at": "t0", "finished_at": "2026-11-02T15:00:00Z", + "exit_code": 0, "refresh_failed": True, "errors": [BCACHEFS_ERR]}} if scenario == "check_failed": return {"running": False, "available": [], "reboot": no_reboot, "last_run": {"action": "check", "started_at": "t0", "finished_at": "t1", "exit_code": 100, "errors": [RESOLVE_ERR]}} @@ -50,7 +70,8 @@ def status(scenario, state): if os.environ.get("CHROMIUM_PATH"): options["executable_path"] = os.environ["CHROMIUM_PATH"] browser = playwright.chromium.launch(**options) - for scenario in ("restart_needed", "up_to_date", "check_failed", "available"): + for scenario in ("restart_needed", "up_to_date", "check_failed", "install_failed_superseded", + "install_failed", "installed_refresh_failed", "available"): context = browser.new_context(ignore_https_errors=os.environ.get("ALLOW_INSECURE_TLS") == "1") context.add_init_script("localStorage.setItem('sn_session','true'); localStorage.setItem('sn_user','qa');") reboots = [] @@ -107,6 +128,19 @@ def respond(route): page.get_by_role("button", name="Check for updates", exact=True).click() expect(page.get_by_text("Checking for updates…", exact=True)).to_be_visible() assert page.get_by_text("Installing updates…", exact=False).count() == 0, "a check was labelled as an install" + elif scenario == "install_failed_superseded": + expect(page.get_by_text("SmoothNAS is up to date.", exact=False)).to_be_visible() + expect(page.get_by_text("automatic updates have installed successfully since (2026-11-02T13:38:27Z)", exact=False)).to_be_visible() + assert page.get_by_text("Installing updates failed", exact=False).count() == 0, "a superseded failure is shown as current" + assert page.locator(".error-msg").count() == 0, "a superseded failure is shown as an error" + elif scenario == "install_failed": + expect(page.get_by_text("Installing updates failed (2026-11-01T15:36:28Z).", exact=True)).to_be_visible() + expect(page.get_by_text(BCACHEFS_ERR, exact=True)).to_be_visible() + assert page.get_by_text("automatic updates have installed", exact=False).count() == 0 + elif scenario == "installed_refresh_failed": + expect(page.get_by_text("Updates were installed (2026-11-02T15:00:00Z), but these package sources could not be refreshed:", exact=True)).to_be_visible() + expect(page.get_by_text(BCACHEFS_ERR, exact=True)).to_be_visible() + assert page.get_by_text("Installing updates failed", exact=False).count() == 0, "a successful install is shown as failed" elif scenario == "check_failed": expect(page.get_by_text("Checking for updates failed", exact=False)).to_be_visible() expect(page.get_by_text(RESOLVE_ERR, exact=True)).to_be_visible() diff --git a/tierd-ui/src/api/api.ts b/tierd-ui/src/api/api.ts index bd65fc3a..35502cdf 100644 --- a/tierd-ui/src/api/api.ts +++ b/tierd-ui/src/api/api.ts @@ -311,6 +311,12 @@ export const api = { getUpdateLog: (offset: number) => apiFetch('GET', `/system/updates/log?offset=${offset}`), // --- Backup --- + getGoogleBackupAccounts: () => apiFetch>('GET', '/backup/google/accounts'), + deleteGoogleBackupAccount: (id: number) => apiFetch('DELETE', `/backup/google/accounts/${id}`), + startGoogleBackupSession: (credentials: { label: string; client_id: string; client_secret: string }) => apiFetch<{ session_id: string; user_code: string; verification_url: string }>('POST', '/backup/google/device/start', credentials), + getGoogleBackupSession: (id: string) => apiFetch<{ status: string }>('GET', `/backup/google/device/${id}`), + cancelGoogleBackupSession: (id: string) => apiFetch('DELETE', `/backup/google/device/${id}`), + prepareGoogleBackupRecovery: (config: any) => apiFetch<{ session_id: string; recovery_kit: string }>('POST', '/backup/google/recovery', config), getBackupConfigs: () => apiFetch('GET', '/backup/configs'), createBackupConfig: (cfg: any) => apiFetch('POST', '/backup/configs', cfg), updateBackupConfig: (id: number, cfg: any) => apiFetch('PUT', `/backup/configs/${id}`, cfg), diff --git a/tierd-ui/src/i18n/locales/en.ts b/tierd-ui/src/i18n/locales/en.ts index 8a66f286..1a6590fb 100644 --- a/tierd-ui/src/i18n/locales/en.ts +++ b/tierd-ui/src/i18n/locales/en.ts @@ -724,6 +724,68 @@ const en: LanguageTranslations = { 'backups.empty': 'No backup configs yet. Add one above.', 'backups.form.editTitle': 'Edit Backup Config', 'backups.form.newTitle': 'New Backup Config', + "backups.google.accounts": "Google accounts", + "backups.google.setup": "Create your own OAuth client of type TVs and Limited Input devices with the Google Drive API enabled.", + "backups.google.console": "Google Cloud Console", + "backups.google.production": "Set the OAuth consent screen to External and publish to Production. Testing tokens expire after 7 days.", + "backups.google.sameProject": "Reuse the same Google Cloud project when reconnecting: another project cannot see existing backups.", + "backups.google.jobs": "{count} jobs", + "backups.google.disconnect": "Disconnect", + "backups.google.disconnectConfirm": "Remove this account from SmoothNAS? Files in Drive remain. You can revoke access in your Google account settings.", + "backups.google.connect": "Connect Google account", + "backups.google.label": "Account label", + "backups.google.clientId": "OAuth client ID", + "backups.google.clientSecret": "OAuth client secret", + "backups.google.approve": "Open Google and enter this code:", + "backups.google.status.pending": "Waiting for approval", + "backups.google.signInFailed": "Google sign-in did not complete. Check credentials and the account label, then connect again.", + "backups.google.method": "Google Drive", + "backups.google.account": "Google account", + "backups.google.selectAccount": "Select a connected account", + "backups.google.excludes": "Exclude patterns (one per line)", + "backups.google.encrypt": "Encrypt files and filenames (fixed after the first run)", + "backups.google.encrypted": "Encrypted", + "backups.google.recoveryWarning": "Keep the recovery kit somewhere other than this NAS. Without it, losing the NAS means losing access to encrypted backups.", + "backups.google.downloadKit": "Download recovery kit", + "backups.google.kitSaved": "I saved the recovery kit somewhere safe", + "backups.schedule.enabled": "Schedule automatically", + "backups.schedule.time": "Time (NAS local timezone)", + "backups.schedule.next": "Next run", + "backups.schedule.mon": "Mon", + "backups.schedule.tue": "Tue", + "backups.schedule.wed": "Wed", + "backups.schedule.thu": "Thu", + "backups.schedule.fri": "Fri", + "backups.schedule.sat": "Sat", + "backups.schedule.sun": "Sun", + "error.backup.invalid_account_id": "invalid account id", + "error.backup.account_unavailable_or_still_in_use": "Account unavailable or still in use", + "error.backup.sign_in_session_expired_connect_again": "Sign-in session expired; connect again", + "error.backup.google_account_and_a_relative_drive_folder_are_required": "Google account and a relative Drive folder are required", + "error.backup.google_account_not_found": "Google account not found", + "error.backup.too_many_recovery_preparations_try_again_later": "Too many recovery preparations; try again later", + "error.backup.label_client_id_and_client_secret_are_required": "Label, client ID and client secret are required", + "error.backup.too_many_sign_in_sessions_cancel_an_existing_attempt": "Too many sign-in sessions; cancel an existing attempt", + "error.backup.google_could_not_start_sign_in_check_the_oauth_client_type_a": "Google could not start sign-in; check the OAuth client type and credentials", + "error.backup.invalid_google_sign_in_response": "Invalid Google sign-in response", + "error.backup.cannot_edit_a_running_backup": "Cannot edit a running backup", + "error.backup.backup_is_already_running": "Backup is already running", + 'backups.google.prepareKit': 'Prepare recovery kit', + "error.backup.schedule_invalid": "schedule requires HH:MM and weekdays between 1 and 127", + "error.backup.google_account_required": "Google Drive requires a Google account", + "error.backup.drive_folder_invalid": "Drive folder must be a nonempty relative path", + "error.backup.local_path_absolute": "local_path must be absolute", + "error.backup.direction_invalid": "direction must be push or pull", + "error.backup.drive_restore_no_delete": "Drive restores never delete local files", + "error.backup.excludes_invalid": "invalid excludes", + "error.backup.drive_fields_invalid": "Google account and encryption require Google Drive", + "error.backup.encrypted_method_fixed": "encrypted Drive jobs cannot change method", + "error.backup.encryption_fixed": "encryption cannot change after the first run", + "error.backup.encrypted_target_fixed": "encrypted jobs keep the account and folder recorded in their recovery kit; create a new job for another destination", + "error.backup.recovery_required": "download and acknowledge the recovery kit before saving", + 'backups.google.fixedTarget': 'The encrypted account and folder stay fixed to match your recovery kit. Create a new job for another destination.', + 'backups.google.excludesHelp': 'For example, /video/** excludes everything in the video folder.', + 'backups.google.folder': 'Drive folder', 'backups.field.direction': 'Direction', 'backups.direction.push': 'Push (local → remote)', 'backups.direction.pull': 'Pull (remote → local)', @@ -1044,6 +1106,8 @@ const en: LanguageTranslations = { 'updates.label.installing': 'Installing updates… This continues even if you leave this page.', 'updates.label.checkFailed': 'Checking for updates failed ({time}).', 'updates.label.installFailed': 'Installing updates failed ({time}).', + 'updates.label.installFailedSuperseded': 'An update started here failed ({failed}); automatic updates have installed successfully since ({time}).', + 'updates.label.installedRefreshFailed': 'Updates were installed ({time}), but these package sources could not be refreshed:', 'updates.label.upToDate': 'SmoothNAS is up to date.', 'updates.label.lastChecked': '(last checked {time})', 'updates.label.available': '{count} update(s) available', diff --git a/tierd-ui/src/i18n/locales/nl.ts b/tierd-ui/src/i18n/locales/nl.ts index f0def0b5..582b8baf 100644 --- a/tierd-ui/src/i18n/locales/nl.ts +++ b/tierd-ui/src/i18n/locales/nl.ts @@ -744,6 +744,68 @@ const nl: LanguageTranslations = { 'backups.empty': 'Nog geen back-upconfiguraties. Voeg er hierboven één toe.', 'backups.form.editTitle': 'Back-upconfiguratie bewerken', 'backups.form.newTitle': 'Nieuwe back-upconfiguratie', + "backups.google.accounts": "Google-accounts", + "backups.google.setup": "Maak een eigen OAuth-client van het type TVs and Limited Input devices en schakel de Google Drive API in.", + "backups.google.console": "Google Cloud Console", + "backups.google.production": "Stel het OAuth-toestemmingsscherm in op External en publiceer naar Production. Testtokens verlopen na 7 dagen.", + "backups.google.sameProject": "Gebruik bij opnieuw verbinden hetzelfde Google Cloud-project: een ander project kan bestaande back-ups niet zien.", + "backups.google.jobs": "{count} taken", + "backups.google.disconnect": "Verbinding verbreken", + "backups.google.disconnectConfirm": "Dit account uit SmoothNAS verwijderen? Bestanden in Drive blijven behouden. Je kunt toegang intrekken in je Google-accountinstellingen.", + "backups.google.connect": "Google-account verbinden", + "backups.google.label": "Accountnaam", + "backups.google.clientId": "OAuth-client-ID", + "backups.google.clientSecret": "OAuth-clientgeheim", + "backups.google.approve": "Open Google en voer deze code in:", + "backups.google.status.pending": "Wachten op toestemming", + "backups.google.signInFailed": "Google-aanmelding is niet voltooid. Controleer de gegevens en accountnaam en verbind opnieuw.", + "backups.google.method": "Google Drive", + "backups.google.account": "Google-account", + "backups.google.selectAccount": "Selecteer een verbonden account", + "backups.google.excludes": "Uitsluitpatronen (één per regel)", + "backups.google.encrypt": "Bestanden en bestandsnamen versleutelen (vast na de eerste uitvoering)", + "backups.google.encrypted": "Versleuteld", + "backups.google.recoveryWarning": "Bewaar de herstelkit buiten deze NAS. Zonder herstelkit zijn versleutelde back-ups na verlies van de NAS niet meer toegankelijk.", + "backups.google.downloadKit": "Herstelkit downloaden", + "backups.google.kitSaved": "Ik heb de herstelkit veilig opgeslagen", + "backups.schedule.enabled": "Automatisch plannen", + "backups.schedule.time": "Tijd (lokale tijdzone van de NAS)", + "backups.schedule.next": "Volgende uitvoering", + "backups.schedule.mon": "Ma", + "backups.schedule.tue": "Di", + "backups.schedule.wed": "Wo", + "backups.schedule.thu": "Do", + "backups.schedule.fri": "Vr", + "backups.schedule.sat": "Za", + "backups.schedule.sun": "Zo", + "error.backup.invalid_account_id": "Ongeldige account-ID", + "error.backup.account_unavailable_or_still_in_use": "Account is niet beschikbaar of nog in gebruik", + "error.backup.sign_in_session_expired_connect_again": "Aanmeldsessie verlopen; verbind opnieuw", + "error.backup.google_account_and_a_relative_drive_folder_are_required": "Een Google-account en relatieve Drive-map zijn vereist", + "error.backup.google_account_not_found": "Google-account niet gevonden", + "error.backup.too_many_recovery_preparations_try_again_later": "Te veel voorbereide herstelkits; probeer het later opnieuw", + "error.backup.label_client_id_and_client_secret_are_required": "Accountnaam, client-ID en clientgeheim zijn vereist", + "error.backup.too_many_sign_in_sessions_cancel_an_existing_attempt": "Te veel aanmeldsessies; annuleer een bestaande poging", + "error.backup.google_could_not_start_sign_in_check_the_oauth_client_type_a": "Google kon de aanmelding niet starten; controleer het OAuth-clienttype en de gegevens", + "error.backup.invalid_google_sign_in_response": "Ongeldig antwoord bij Google-aanmelding", + "error.backup.cannot_edit_a_running_backup": "Een actieve back-up kan niet worden gewijzigd of verwijderd", + "error.backup.backup_is_already_running": "Back-up is al actief", + 'backups.google.prepareKit': 'Herstelkit voorbereiden', + "error.backup.schedule_invalid": "Kies een geldige tijd (HH:MM) en minstens één weekdag", + "error.backup.google_account_required": "Google Drive vereist een Google-account", + "error.backup.drive_folder_invalid": "De Drive-map moet een niet-leeg relatief pad zijn", + "error.backup.local_path_absolute": "Het lokale pad moet absoluut zijn", + "error.backup.direction_invalid": "De richting moet push of pull zijn", + "error.backup.drive_restore_no_delete": "Drive-herstel verwijdert nooit lokale bestanden", + "error.backup.excludes_invalid": "Ongeldige uitsluitpatronen", + "error.backup.drive_fields_invalid": "Een Google-account en versleuteling vereisen Google Drive", + "error.backup.encrypted_method_fixed": "De methode van een versleutelde Drive-taak kan niet worden gewijzigd", + "error.backup.encryption_fixed": "Versleuteling kan na de eerste uitvoering niet worden gewijzigd", + "error.backup.encrypted_target_fixed": "Het account en de map van een versleutelde taak liggen vast in de herstelkit; maak een nieuwe taak voor een andere bestemming", + "error.backup.recovery_required": "Download de herstelkit en bevestig dat deze veilig is opgeslagen", + 'backups.google.fixedTarget': 'Het versleutelde account en de map liggen vast in de herstelkit. Maak een nieuwe taak voor een andere bestemming.', + 'backups.google.excludesHelp': 'Bijvoorbeeld: /video/** sluit alles in de map video uit.', + 'backups.google.folder': 'Drive-map', 'backups.field.direction': 'Richting', 'backups.direction.push': 'Push (lokaal → remote)', 'backups.direction.pull': 'Pull (remote → lokaal)', @@ -1064,6 +1126,8 @@ const nl: LanguageTranslations = { 'updates.label.installing': 'Updates worden geïnstalleerd… Dit gaat door, ook als je deze pagina verlaat.', 'updates.label.checkFailed': 'Zoeken naar updates mislukt ({time}).', 'updates.label.installFailed': 'Installeren van updates mislukt ({time}).', + 'updates.label.installFailedSuperseded': 'Een hier gestarte update is mislukt ({failed}); automatische updates zijn sindsdien wel geslaagd ({time}).', + 'updates.label.installedRefreshFailed': 'Updates zijn geïnstalleerd ({time}), maar deze pakketbronnen konden niet worden vernieuwd:', 'updates.label.upToDate': 'SmoothNAS is up-to-date.', 'updates.label.lastChecked': '(laatst gecontroleerd {time})', 'updates.label.available': '{count} update(s) beschikbaar', diff --git a/tierd-ui/src/pages/Backups/Backups.tsx b/tierd-ui/src/pages/Backups/Backups.tsx index cfc6a63e..5411bde0 100644 --- a/tierd-ui/src/pages/Backups/Backups.tsx +++ b/tierd-ui/src/pages/Backups/Backups.tsx @@ -1,14 +1,16 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useI18n } from '@rakuensoftware/smoothgui'; import { api } from '../../api/api'; import { useToast } from '../../contexts/ToastContext'; import ConfirmDialog from '../../components/ConfirmDialog/ConfirmDialog'; import NumberInput from '../../components/NumberInput/NumberInput'; +import GoogleAccounts, { type GoogleAccount } from './GoogleAccounts'; + interface BackupConfig { id: number; name: string; - target_type: 'nfs' | 'smb'; + target_type: 'nfs' | 'smb' | 'gdrive'; host: string; share: string; smb_user: string; @@ -18,11 +20,19 @@ interface BackupConfig { local_path: string; remote_path: string; direction: 'push' | 'pull'; - method: 'cp' | 'rsync'; + method: 'cp' | 'rsync' | 'rclone'; parallelism: number; use_ssh: boolean; compress: boolean; delete_mode: boolean; + google_account_id: number | null; + excludes: string; + encrypt: boolean; + has_run: boolean; + schedule_enabled: boolean; + schedule_time: string; + schedule_days: number; + next_run_at: string; created_at: string; } @@ -41,8 +51,14 @@ interface BackupRun { } const EMPTY_FORM = { + google_account_id: null as number | null, + excludes: '', + encrypt: false, + schedule_enabled: false, + schedule_time: '02:00', + schedule_days: 127, name: '', - target_type: 'nfs' as 'nfs' | 'smb', + target_type: 'nfs' as 'nfs' | 'smb' | 'gdrive', host: '', share: '', smb_user: '', @@ -52,7 +68,7 @@ const EMPTY_FORM = { local_path: '', remote_path: '', direction: 'push' as 'push' | 'pull', - method: 'rsync' as 'cp' | 'rsync', + method: 'rsync' as 'cp' | 'rsync' | 'rclone', parallelism: 4, use_ssh: false, compress: false, @@ -85,6 +101,10 @@ export default function Backups() { const { t } = useI18n(); const { success: toastSuccess, error: toastError } = useToast(); const [configs, setConfigs] = useState([]); + const [googleAccounts, setGoogleAccounts] = useState([]); + const refreshGoogle = useCallback(() => { api.getGoogleBackupAccounts().then(setGoogleAccounts).catch(e => toastError(e.message)); }, [toastError]); + const [recovery, setRecovery] = useState<{ session_id: string; recovery_kit: string } | null>(null); + const [recoveryConfirmed, setRecoveryConfirmed] = useState(false); const [loading, setLoading] = useState(true); const [showForm, setShowForm] = useState(false); const [form, setForm] = useState({ ...EMPTY_FORM }); @@ -105,13 +125,30 @@ export default function Backups() { // Capture pollRefs.current at effect-setup time so the cleanup // sees the same map even if pollRefs is reassigned later. const intervals = pollRefs.current; + // Discover runs started by the scheduler while this page stays open. + let disposed = false; + const schedulePoll = setInterval(() => { + Promise.all([api.getBackupConfigs(), api.listBackupRuns()]).then(([cfgs, active]) => { + if (disposed) return; + setConfigs(cfgs); + for (const run of active as BackupRun[]) { + if (!intervals[run.config_id]) { + setRuns(previous => ({ ...previous, [run.config_id]: run })); + pollRun(run.config_id, run.id); + } + } + }).catch(() => {}); + }, 10000); return () => { + disposed = true; + clearInterval(schedulePoll); Object.values(intervals).forEach(clearInterval); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); function load() { + refreshGoogle(); setLoading(true); Promise.all([ api.getBackupConfigs(), @@ -132,6 +169,7 @@ export default function Backups() { } function setField(key: keyof typeof EMPTY_FORM, value: string | number | boolean) { + setRecovery(null); setRecoveryConfirmed(false); setForm(f => ({ ...f, [key]: value })); } @@ -144,11 +182,18 @@ export default function Backups() { setFormError(''); setSaving(true); try { + if (form.encrypt && !configs.find(c => c.id === editingId)?.encrypt && !recovery) { + setRecovery(await api.prepareGoogleBackupRecovery(form)); + setRecoveryConfirmed(false); + return; + } + const payload = { ...form, recovery_session: recovery?.session_id || '', recovery_confirmed: recoveryConfirmed }; if (editingId != null) { - await api.updateBackupConfig(editingId, form); + await api.updateBackupConfig(editingId, payload); } else { - await api.createBackupConfig(form); + await api.createBackupConfig(payload); } + setRecovery(null); setRecoveryConfirmed(false); setForm({ ...EMPTY_FORM }); setShowForm(false); setEditingId(null); @@ -164,7 +209,14 @@ export default function Backups() { function startEdit(cfg: BackupConfig) { // Passwords are never returned by the API — leave blank. The backend // treats blank smb_pass / ssh_pass on update as "keep existing secret". + setRecovery(null); setRecoveryConfirmed(false); setForm({ + google_account_id: cfg.google_account_id, + excludes: cfg.excludes, + encrypt: cfg.encrypt, + schedule_enabled: cfg.schedule_enabled, + schedule_time: cfg.schedule_time || '02:00', + schedule_days: cfg.schedule_days, name: cfg.name, target_type: cfg.target_type, host: cfg.host, @@ -188,6 +240,7 @@ export default function Backups() { } function cancelForm() { + setRecovery(null); setRecoveryConfirmed(false); setShowForm(false); setEditingId(null); setForm({ ...EMPTY_FORM }); @@ -230,17 +283,19 @@ export default function Backups() { } function pollRun(cfgId: number, runId: number) { - if (pollRefs.current[cfgId]) clearInterval(pollRefs.current[cfgId]); + if (pollRefs.current[cfgId]) { clearInterval(pollRefs.current[cfgId]); delete pollRefs.current[cfgId]; } stallRefs.current[cfgId] = { lastHealthyAt: Date.now(), notified: false }; pollRefs.current[cfgId] = setInterval(() => { api.getBackupRun(runId).then((run: BackupRun) => { setRuns(prev => ({ ...prev, [cfgId]: run })); if (run.status === 'completed') { clearInterval(pollRefs.current[cfgId]); + delete pollRefs.current[cfgId]; delete stallRefs.current[cfgId]; toastSuccess(t('backups.toast.complete')); } else if (run.status === 'failed') { clearInterval(pollRefs.current[cfgId]); + delete pollRefs.current[cfgId]; delete stallRefs.current[cfgId]; if (run.error && run.error !== 'Cancelled') { toastError(t('backups.toast.stopped', { err: run.error })); @@ -259,7 +314,10 @@ export default function Backups() { } } } - }).catch(() => clearInterval(pollRefs.current[cfgId])); + }).catch(() => { + clearInterval(pollRefs.current[cfgId]); + delete pollRefs.current[cfgId]; + }); }, 500); } @@ -286,6 +344,7 @@ export default function Backups() { + {showForm && (

{editingId != null ? t('backups.form.editTitle') : t('backups.form.newTitle')}

@@ -295,13 +354,14 @@ export default function Backups() { setField('name', e.target.value)} />
)} -
+ {form.method !== 'rclone' &&
@@ -366,7 +426,7 @@ export default function Backups() { onChange={e => setField('share', e.target.value)} /> -
+
} {form.method === 'rsync' && form.use_ssh && (
@@ -434,15 +494,45 @@ export default function Backups() { /> )} -
+ {form.method === 'rclone' && <> +
+ +