diff --git a/README.md b/README.md index 5db127d..3e391ab 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,15 @@ docker run -d --name backupx -p 8340:8340 -v backupx-data:/app/data awuqing/back # Or prebuilt archive curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-linux-amd64.tar.gz tar xzf backupx-*.tar.gz && cd backupx-* && sudo ./install.sh + +# Or build and install on bare metal without Docker +git clone https://github.com/Awuqing/BackupX.git && cd BackupX +make build && sudo ./deploy/install.sh ``` For ARM64 hosts, use `backupx-linux-arm64.tar.gz`. The archive contains `backupx`, `web/`, `config.example.yaml`, and `install.sh`; run `install.sh` from the extracted directory. -Open `http://your-server:8340`, create the admin account, then follow the [5-minute Quick Start](https://awuqing.github.io/BackupX/docs/getting-started/quick-start). +Open `http://your-server:8340`, choose English or Chinese on the setup screen, create the first administrator account, then follow the [5-minute Quick Start](https://awuqing.github.io/BackupX/docs/getting-started/quick-start). ## Documentation diff --git a/README.zh-CN.md b/README.zh-CN.md index 661a9b0..e6a0b27 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -60,11 +60,15 @@ docker run -d --name backupx -p 8340:8340 -v backupx-data:/app/data awuqing/back # 或使用预编译包 curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-linux-amd64.tar.gz tar xzf backupx-*.tar.gz && cd backupx-* && sudo ./install.sh + +# 或从源码构建并裸机安装（无需 Docker） +git clone https://github.com/Awuqing/BackupX.git && cd BackupX +make build && sudo ./deploy/install.sh ``` ARM64 主机请下载 `backupx-linux-arm64.tar.gz`。预编译包内包含 `backupx`、`web/`、`config.example.yaml` 和 `install.sh`，请在解压后的目录内执行 `install.sh`。 -打开 `http://your-server:8340`，创建管理员账户，按 [5 分钟快速开始](https://awuqing.github.io/BackupX/zh-Hans/docs/getting-started/quick-start) 完成首次备份。 +打开 `http://your-server:8340`，在初始化页选择中文或 English 并创建首个管理员账户，按 [5 分钟快速开始](https://awuqing.github.io/BackupX/zh-Hans/docs/getting-started/quick-start) 完成首次备份。 ## 文档 diff --git a/deploy/install.sh b/deploy/install.sh index d451ae7..da068a1 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -14,7 +14,13 @@ if [ -f "$SCRIPT_DIR/backupx" ] && [ -d "$SCRIPT_DIR/web" ]; then CONFIG_TEMPLATE="${CONFIG_TEMPLATE:-$SCRIPT_DIR/config.example.yaml}" NGINX_SOURCE="${NGINX_SOURCE:-$SCRIPT_DIR/nginx.conf}" else - BIN_SOURCE="${BIN_SOURCE:-$PROJECT_ROOT/server/backupx}" + SOURCE_BIN_DEFAULT="$PROJECT_ROOT/server/bin/backupx" + # Keep compatibility with contributors who built the historical path by + # hand, while matching the canonical `make build` output first. + if [ ! -f "$SOURCE_BIN_DEFAULT" ] && [ -f "$PROJECT_ROOT/server/backupx" ]; then + SOURCE_BIN_DEFAULT="$PROJECT_ROOT/server/backupx" + fi + BIN_SOURCE="${BIN_SOURCE:-$SOURCE_BIN_DEFAULT}" WEB_SOURCE="${WEB_SOURCE:-$PROJECT_ROOT/web/dist}" CONFIG_TEMPLATE="${CONFIG_TEMPLATE:-$PROJECT_ROOT/server/config.example.yaml}" NGINX_SOURCE="${NGINX_SOURCE:-$PROJECT_ROOT/deploy/nginx.conf}" @@ -27,8 +33,9 @@ if [ "$(id -u)" -ne 0 ]; then fi if [ ! -f "$BIN_SOURCE" ]; then - echo "未找到后端二进制：$BIN_SOURCE" >&2 - echo "源码树安装请先执行：cd \"$PROJECT_ROOT/server\" && go build -o backupx ./cmd/backupx" >&2 + echo "Backend binary not found / 未找到后端二进制：$BIN_SOURCE" >&2 + echo "源码树安装请先在仓库根目录执行 make build（产物：server/bin/backupx）。" >&2 + echo "For a source install, run 'make build' in the repository root first." >&2 echo "发布包安装请确认当前目录包含 ./backupx、./web 和 ./install.sh。" >&2 exit 1 fi @@ -92,6 +99,41 @@ fi systemctl daemon-reload systemctl enable --now "$SERVICE_NAME" +# systemctl may return before the process has opened its HTTP listener. Verify +# the same unauthenticated endpoint used by the first-administrator screen so a +# broken bare-metal install cannot print a false success message. +HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8340/api/auth/setup/status}" +READY=0 +ATTEMPT=1 +while [ "$ATTEMPT" -le 30 ]; do + if systemctl is-active --quiet "$SERVICE_NAME"; then + if command -v curl >/dev/null 2>&1; then + if curl -fsS --max-time 2 "$HEALTH_URL" >/dev/null 2>&1; then + READY=1 + break + fi + elif command -v wget >/dev/null 2>&1; then + if wget -q -T 2 -O /dev/null "$HEALTH_URL"; then + READY=1 + break + fi + else + echo "Warning / 警告：未找到 curl 或 wget，仅验证 systemd 服务状态。" >&2 + READY=1 + break + fi + fi + ATTEMPT=$((ATTEMPT + 1)) + sleep 1 +done + +if [ "$READY" -ne 1 ]; then + echo "BackupX did not become ready at $HEALTH_URL / 服务未通过就绪检查。" >&2 + systemctl status "$SERVICE_NAME" --no-pager >&2 || true + journalctl -u "$SERVICE_NAME" -n 50 --no-pager >&2 || true + exit 1 +fi + if [ -d "/etc/nginx/conf.d" ] && [ -f "$NGINX_SOURCE" ]; then install -m 0644 "$NGINX_SOURCE" "/etc/nginx/conf.d/$SERVICE_NAME.conf" if command -v nginx >/dev/null 2>&1; then @@ -111,6 +153,11 @@ cat <:8340 +首次访问 / First sign-in: + 1. 打开上面的地址，并可在登录页右上角选择 中文 或 English。 + 2. 页面显示“系统初始化 / System setup”时，创建首个管理员用户名和密码。 + 3. 如果未显示初始化表单，请先检查：$HEALTH_URL + （如已安装 nginx，脚本会自动写入反向代理配置，可继续用 80 端口访问。） 排查：若服务未监听端口，请查看日志： diff --git a/docs-site/docs/deployment/bare-metal.md b/docs-site/docs/deployment/bare-metal.md index 2200f56..ee81c82 100644 --- a/docs-site/docs/deployment/bare-metal.md +++ b/docs-site/docs/deployment/bare-metal.md @@ -10,20 +10,21 @@ description: systemd + Nginx deployment from the prebuilt release tarball or sou ```bash # Download the matching tarball -curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-v1.6.0-linux-amd64.tar.gz +curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-linux-amd64.tar.gz # Extract and install -tar xzf backupx-v*-linux-amd64.tar.gz && cd backupx-* +tar xzf backupx-linux-amd64.tar.gz && cd backupx-*-linux-amd64 sudo ./install.sh ``` The installer performs these steps automatically: 1. Creates a system user `backupx` -2. Copies the binary to `/opt/backupx/` -3. Generates a default `config.yaml` with safe JWT/encryption secrets +2. Copies the binary to `/opt/backupx/bin/backupx` and the web console to `/opt/backupx/web` +3. Installs the default configuration at `/etc/backupx/config.yaml` 4. Installs `backupx.service` (systemd), enabled at boot 5. (Optional) installs an Nginx site file — see [Nginx Reverse Proxy](./nginx) +6. Verifies the first-setup API before reporting success For multi-node clusters, edit `/etc/backupx/config.yaml` after installation and set the Master URL that remote Agents can reach: @@ -63,11 +64,13 @@ After=network.target [Service] Type=simple User=backupx +Group=backupx WorkingDirectory=/opt/backupx -ExecStart=/opt/backupx/backupx --config /opt/backupx/config.yaml +ExecStart=/opt/backupx/bin/backupx -config /etc/backupx/config.yaml Restart=on-failure -RestartSec=5s -LimitNOFILE=65536 +RestartSec=5 +NoNewPrivileges=true +LimitNOFILE=65535 [Install] WantedBy=multi-user.target @@ -79,17 +82,20 @@ Typical operations: sudo systemctl status backupx sudo journalctl -u backupx -f # live logs sudo systemctl restart backupx +curl -fsS http://127.0.0.1:8340/api/auth/setup/status ``` +Open `http://your-server:8340`, switch to English if desired, and create the first administrator on the **System setup** screen. For a custom listen port, run the installer with a matching `HEALTH_URL`. + ## Password reset If the admin password is lost: ```bash -/opt/backupx/backupx reset-password \ +/opt/backupx/bin/backupx reset-password \ --username admin \ --password 'newpass123' \ - --config /opt/backupx/config.yaml + --config /etc/backupx/config.yaml ``` Docker equivalent: diff --git a/docs-site/docs/features/backup-types.md b/docs-site/docs/features/backup-types.md index 5285c20..f41b946 100644 --- a/docs-site/docs/features/backup-types.md +++ b/docs-site/docs/features/backup-types.md @@ -12,12 +12,22 @@ When a task is routed to a remote Agent, the source tools and paths are resolved ## File / Directory -Tars (and optionally gzips) one or more filesystem paths. +File tasks offer three backup modes: + +- **Full archive** — writes a self-contained tar artifact on every run +- **Differential archive** — writes only changes since the current full baseline and periodically refreshes that baseline +- **CDC repository** — splits content with stable 512 KiB / 1 MiB / 4 MiB boundaries, stores new chunks in immutable 32 MiB packs, and writes a small snapshot manifest for each run + +The CDC repository deduplicates identical content across files and snapshots. Restore, selective restore, verification, download-as-tar, retention, and garbage collection all resolve data through the repository index. Compression and encryption are applied per chunk; encrypted repositories use keyed chunk IDs so plaintext hashes are not exposed. + +Repository mode currently uses a single-writer index and therefore runs on the Master only. To keep repository copies on multiple backends, select multiple primary storage targets on the task. Object-level replication is intentionally disabled because a snapshot manifest without its shared packs and indexes is not a complete backup. + +Common file-task options: - **Source** accepts multiple paths — one per line in the UI - **Exclude patterns** accept gitignore-style globs - Supports following symlinks, preserving permissions -- Output is a single `.tar` or `.tar.gz` artifact +- Full and differential modes output `.tar`, `.tar.gz`, or `.tar.zst` artifacts ## MySQL diff --git a/docs-site/docs/features/multi-node.md b/docs-site/docs/features/multi-node.md index 0cfc477..b47346b 100644 --- a/docs-site/docs/features/multi-node.md +++ b/docs-site/docs/features/multi-node.md @@ -26,6 +26,27 @@ BackupX supports Master-Agent mode: backup tasks can be routed to specific nodes - **Execution** — Agent reuses the same BackupRunner (file / mysql / postgresql / sqlite / saphana) and uploads directly to storage - **Security** — Each node has its own token; the Agent never holds the Master's JWT secret or AES-256 key +## Centralize backups from servers B/C/D into storage M + +Use the Master as the control plane and register every source server as an Agent. A task's **Source server** determines where paths and database tools are resolved; its **Storage targets** determine where the resulting artifact is retained. + +BackupX chooses the data path per target: + +| Destination | Data path | +| --- | --- | +| S3, WebDAV, FTP, cloud drive, or another network backend | Agent streams directly to the destination | +| `local_disk` with **Relay remote backups through Master** enabled (for example storage server M mounted through NFS) | Agent streams through the authenticated Master API; Master writes to its configured local path | + +The relay is streaming: the Master does not create a second temporary copy of the entire artifact. The reverse path is used when restoring a Master-local artifact back to its source Agent. Use HTTPS whenever Agent traffic crosses an untrusted network. + +To configure the common `A → {B,C,D} → M` topology: + +1. Run BackupX Master on A and mount M on A if M is exposed as NFS or another filesystem. +2. Create a `local_disk` target for that mount and keep **Relay remote backups through Master** enabled, or create an S3/WebDAV target exposed by M. Existing local-disk targets keep their prior Agent-local behavior until this switch is enabled. +3. Install one Agent on B, C, and D from **Node Management**. +4. Create a backup task for each source, choose B/C/D under **Source server**, browse that server's paths, and select M as the storage target. A source-server pool label can route identical tasks dynamically. +5. Verify the per-target result in the backup record. For a Master-local target, the record reports transfer mode `master_relay`; network backends remain `direct`. + ## Walkthrough ### 0. Set the Master URL for production clusters @@ -78,7 +99,7 @@ In Step 1 choose "Batch" and paste node names (one per line, max 50). Step 3 sho ### 5. Route a task to the node -In the **Backup Tasks** page, pick the target node when creating the task. When the task runs: +In the **Backup Tasks** page, pick the source server when creating the task. When the task runs: - Local (`nodeId=0`) → Master executes in-process - Remote node → Master enqueues the command → Agent claims → Agent runs locally → uploads → reports back diff --git a/docs-site/docs/features/storage-backends.md b/docs-site/docs/features/storage-backends.md index 731526b..6e977cb 100644 --- a/docs-site/docs/features/storage-backends.md +++ b/docs-site/docs/features/storage-backends.md @@ -19,7 +19,9 @@ BackupX aims to accept any place you'd want to drop a backup file. | **Google Drive** | Client ID/Secret + OAuth authorization | | **WebDAV** | URL + username/password | | **FTP / FTPS** | Host + port + username/password | -| **Local disk** | Target directory (absolute path) | +| **Local disk** | Target directory (absolute path) + optional Master relay for remote Agents | + +New local-disk targets enable **Relay remote backups through Master** by default. This makes the configured path belong to the Master, so a storage server mounted there can collect backups from many source Agents. Turn the switch off when the path intentionally belongs to each Agent. Existing targets retain their previous Agent-local behavior until explicitly changed. ## Rclone backends diff --git a/docs-site/docs/getting-started/installation.md b/docs-site/docs/getting-started/installation.md index 32f0c85..42bec64 100644 --- a/docs-site/docs/getting-started/installation.md +++ b/docs-site/docs/getting-started/installation.md @@ -55,10 +55,11 @@ sudo ./install.sh # creates system user, installs to /opt/backupx, sets u The installer: 1. Creates a `backupx` system user -2. Installs binary to `/opt/backupx/backupx` -3. Creates `/opt/backupx/config.yaml` with safe defaults +2. Installs the binary to `/opt/backupx/bin/backupx` and the web console to `/opt/backupx/web` +3. Creates `/etc/backupx/config.yaml` with safe defaults 4. Installs and enables the `backupx.service` systemd unit 5. (Optional) Configures an Nginx reverse proxy +6. Waits for `/api/auth/setup/status`; if startup fails, prints systemd diagnostics and exits non-zero ## From source @@ -67,16 +68,17 @@ Requires Go ≥ 1.25 and Node.js ≥ 20. ```bash git clone https://github.com/Awuqing/BackupX.git && cd BackupX make build -# or, for builds behind the great firewall -make docker-cn +sudo ./deploy/install.sh ``` After `make build`, the binary is at `server/bin/backupx` and the built web UI is at `web/dist/`. +The installer consumes those exact paths, so no Docker runtime is required. If an existing configuration uses a non-default port, set `HEALTH_URL` for the readiness check, for example `sudo HEALTH_URL=http://127.0.0.1:9000/api/auth/setup/status ./deploy/install.sh`. ## Verify the install ```bash -backupx --version # e.g. v1.6.0 +/opt/backupx/bin/backupx --version +curl -fsS http://127.0.0.1:8340/api/auth/setup/status ``` -Then open `http://your-server:8340` to see the initial admin setup screen. +Then open `http://your-server:8340`. Choose **English** or **中文** in the upper-right corner. A fresh database shows **System setup**, where you create the first administrator username and password. If that form does not appear, retry the status request above before attempting to sign in. diff --git a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/deployment/bare-metal.md b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/deployment/bare-metal.md index 60e49ba..f58191e 100644 --- a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/deployment/bare-metal.md +++ b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/deployment/bare-metal.md @@ -10,20 +10,21 @@ description: 从预编译包或源码部署 BackupX（systemd + Nginx）。 ```bash # 下载对应平台的压缩包 -curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-v1.6.0-linux-amd64.tar.gz +curl -LO https://github.com/Awuqing/BackupX/releases/latest/download/backupx-linux-amd64.tar.gz # 解压并安装 -tar xzf backupx-v*-linux-amd64.tar.gz && cd backupx-* +tar xzf backupx-linux-amd64.tar.gz && cd backupx-*-linux-amd64 sudo ./install.sh ``` 安装脚本自动完成以下步骤： 1. 创建系统用户 `backupx` -2. 复制二进制到 `/opt/backupx/` -3. 生成默认 `config.yaml`（含安全的 JWT/加密密钥） +2. 复制二进制到 `/opt/backupx/bin/backupx`，并把 Web 控制台复制到 `/opt/backupx/web` +3. 把默认配置安装到 `/etc/backupx/config.yaml` 4. 安装并启用 `backupx.service` systemd 单元 5. （可选）生成 Nginx 站点配置 — 参见 [Nginx 反向代理](./nginx) +6. 验证首次初始化接口就绪后才报告安装成功 如果要部署多节点集群，安装后请编辑 `/etc/backupx/config.yaml`，设置远程 Agent 可访问到的 Master URL： @@ -63,11 +64,13 @@ After=network.target [Service] Type=simple User=backupx +Group=backupx WorkingDirectory=/opt/backupx -ExecStart=/opt/backupx/backupx --config /opt/backupx/config.yaml +ExecStart=/opt/backupx/bin/backupx -config /etc/backupx/config.yaml Restart=on-failure -RestartSec=5s -LimitNOFILE=65536 +RestartSec=5 +NoNewPrivileges=true +LimitNOFILE=65535 [Install] WantedBy=multi-user.target @@ -79,17 +82,20 @@ WantedBy=multi-user.target sudo systemctl status backupx sudo journalctl -u backupx -f # 实时日志 sudo systemctl restart backupx +curl -fsS http://127.0.0.1:8340/api/auth/setup/status ``` +访问 `http://your-server:8340`，可按需切换到 English，然后在“系统初始化 / System setup”页面创建首个管理员。若监听端口不是默认值，请为安装脚本传入对应的 `HEALTH_URL`。 + ## 密码重置 忘记管理员密码时： ```bash -/opt/backupx/backupx reset-password \ +/opt/backupx/bin/backupx reset-password \ --username admin \ --password 'newpass123' \ - --config /opt/backupx/config.yaml + --config /etc/backupx/config.yaml ``` Docker 等效命令： diff --git a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/backup-types.md b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/backup-types.md index df97237..bd6042e 100644 --- a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/backup-types.md +++ b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/backup-types.md @@ -12,12 +12,22 @@ BackupX 支持五种内置备份类型，类型决定了用哪个 runner 执行 ## 文件 / 目录 -打包（可选 gzip）一个或多个文件系统路径。 +文件任务提供三种备份模式： + +- **全量归档** — 每次生成一份可独立使用的 tar 产物 +- **差异归档** — 只保存相对当前全量基线的变化，并按周期刷新全量基线 +- **CDC 去重仓库** — 按稳定的 512 KiB / 1 MiB / 4 MiB 内容边界切块，将新块合并到不可变的 32 MiB pack，每次运行只新增一份小型快照清单 + +CDC 仓库会在不同文件、不同快照之间复用相同内容。完整恢复、选择性恢复、完整性校验、下载为 tar、保留策略和垃圾回收都通过仓库索引定位分块。压缩与加密按块执行；启用加密时使用带密钥的块 ID，不暴露明文哈希。 + +当前仓库索引采用单写者模型，因此 CDC 模式仅在 Master 本机执行。如需保存多份完整仓库，请在任务中直接多选主存储目标。对象级副本复制会被禁用，因为只有快照清单、没有共享 pack 与索引并不是完整备份。 + +文件任务的通用选项： - **源路径** 支持多个（UI 中每行一个） - **排除模式** 支持 gitignore 风格的通配符 - 可选跟随符号链接、保留权限 -- 输出单个 `.tar` 或 `.tar.gz` +- 全量与差异模式输出 `.tar`、`.tar.gz` 或 `.tar.zst` ## MySQL diff --git a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/multi-node.md b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/multi-node.md index 9b5a540..7ad1032 100644 --- a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/multi-node.md +++ b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/multi-node.md @@ -26,6 +26,27 @@ BackupX 支持 Master-Agent 模式：备份任务可以指定在哪个节点执 - **执行** — Agent 复用 BackupRunner（file / mysql / postgresql / sqlite / saphana）并直接上传到存储 - **安全** — 每个节点独立 Token；Agent 不持有 Master 的 JWT 密钥或 AES-256 加密密钥 +## 把 B/C/D 服务器集中备份到 M + +Master 作为控制面，每台源服务器安装一个 Agent。任务里的 **源服务器** 决定源路径和数据库工具在哪台机器解析，**存储目标** 决定备份产物最终保留在哪里。 + +BackupX 会根据目标类型选择数据路径： + +| 目标 | 数据路径 | +| --- | --- | +| S3、WebDAV、FTP、云盘或其他网络后端 | Agent 直接流式上传到目标 | +| 启用 **远程备份经 Master 中转** 的 `local_disk`（例如通过 NFS 挂载的存储服务器 M） | Agent 通过认证后的 Master API 流式中转，由 Master 写入配置目录 | + +中转过程不会在 Master 上额外落一份完整临时文件。把 Master 本地磁盘中的备份恢复到源 Agent 时会走反向流式通道。Agent 与 Master 之间跨越不可信网络时必须配置 HTTPS。 + +典型的 `A → {B,C,D} → M` 拓扑按以下步骤配置： + +1. 在 A 运行 BackupX Master；如果 M 以 NFS 等文件系统提供存储，先把 M 挂载到 A。 +2. 为该挂载点创建 `local_disk` 目标并保持 **远程备份经 Master 中转** 开启；如果 M 提供 S3/WebDAV，也可直接创建对应网络目标。升级前已有的本地磁盘目标会继续沿用 Agent 本机落盘，手动开启该选项后才切换到中央目录。 +3. 从 **节点管理** 分别在 B、C、D 安装 Agent。 +4. 为每台源服务器创建任务，在 **源服务器** 选择 B/C/D，浏览该服务器的路径，再把 M 选为存储目标。相同任务也可用源服务器池标签动态调度。 +5. 在备份记录中检查逐目标结果。Master 本地磁盘目标会记录 `master_relay` 中转模式，网络后端仍为 `direct` 直传。 + ## 一键部署步骤 ### 0. 为生产集群设置 Master 对外 URL @@ -78,7 +99,7 @@ Docker 模式使用同一组环境变量约定：`BACKUPX_AGENT_MASTER`、`BACKU ### 5. 把任务路由到该节点 -在 **备份任务** 页面新建任务时选择对应节点。任务触发时： +在 **备份任务** 页面新建任务时选择对应源服务器。任务触发时： - 本机 / 未指定（`nodeId=0`）：Master 进程内直接执行 - 远程节点：Master 写入命令队列 → Agent 拉取 → Agent 本地执行 → 上传 → 回报 diff --git a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/storage-backends.md b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/storage-backends.md index cdd25ac..b3f7895 100644 --- a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/storage-backends.md +++ b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/features/storage-backends.md @@ -19,7 +19,9 @@ BackupX 的目标是接入任何你想放置备份文件的地方。 | **Google Drive** | Client ID/Secret + OAuth 授权 | | **WebDAV** | 地址 + 用户名/密码 | | **FTP / FTPS** | 主机 + 端口 + 用户名/密码 | -| **本地磁盘** | 目标目录（绝对路径） | +| **本地磁盘** | 目标目录（绝对路径）+ 可选的远程 Agent 经 Master 中转 | + +新建本地磁盘目标默认开启 **远程备份经 Master 中转**。开启时，配置目录属于 Master，挂载到 Master 的存储服务器可集中接收多台源 Agent 的备份；如果该路径本就属于各 Agent，请关闭此选项。升级前已有目标保持原来的 Agent 本机落盘行为，只有显式开启后才会切换。 ## Rclone 后端 diff --git a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/getting-started/installation.md b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/getting-started/installation.md index 3ac0208..3fbac14 100644 --- a/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/getting-started/installation.md +++ b/docs-site/i18n/zh-CN/docusaurus-plugin-content-docs/current/getting-started/installation.md @@ -55,10 +55,11 @@ sudo ./install.sh # 创建系统用户、安装到 /opt/backupx、配置 安装脚本会自动： 1. 创建 `backupx` 系统用户 -2. 安装二进制到 `/opt/backupx/backupx` -3. 生成 `/opt/backupx/config.yaml`（含安全默认值） +2. 安装二进制到 `/opt/backupx/bin/backupx`，并把 Web 控制台安装到 `/opt/backupx/web` +3. 生成 `/etc/backupx/config.yaml`（含安全默认值） 4. 注册并启用 `backupx.service` systemd 单元 5. （可选）配置 Nginx 反向代理 +6. 等待 `/api/auth/setup/status` 就绪；启动失败时输出 systemd 诊断并返回非零状态 ## 从源码构建 @@ -67,16 +68,17 @@ sudo ./install.sh # 创建系统用户、安装到 /opt/backupx、配置 ```bash git clone https://github.com/Awuqing/BackupX.git && cd BackupX make build -# 或使用国内镜像加速构建 Docker -make docker-cn +sudo ./deploy/install.sh ``` `make build` 完成后，二进制位于 `server/bin/backupx`，构建好的 Web UI 位于 `web/dist/`。 +安装脚本会直接使用这两个路径，不需要 Docker 运行时。如果已有配置修改了默认端口，可覆盖就绪检查地址，例如：`sudo HEALTH_URL=http://127.0.0.1:9000/api/auth/setup/status ./deploy/install.sh`。 ## 验证安装 ```bash -backupx --version # 输出如 v1.6.0 +/opt/backupx/bin/backupx --version +curl -fsS http://127.0.0.1:8340/api/auth/setup/status ``` -打开浏览器访问 `http://your-server:8340`，会进入初始化管理员账户页面。 +打开浏览器访问 `http://your-server:8340`，可在右上角选择 **中文** 或 **English**。全新数据库会显示“系统初始化 / System setup”，在这里创建首个管理员用户名和密码。如果没有出现初始化表单，请先重试上面的状态接口，不要直接尝试登录。 diff --git a/server/internal/agent/client.go b/server/internal/agent/client.go index 08751ea..64ba166 100644 --- a/server/internal/agent/client.go +++ b/server/internal/agent/client.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -125,10 +126,11 @@ type TaskSpec struct { // StorageTargetConfig 与 service.AgentStorageTargetConfig 对齐 type StorageTargetConfig struct { - ID uint `json:"id"` - Type string `json:"type"` - Name string `json:"name"` - Config json.RawMessage `json:"config"` + ID uint `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Config json.RawMessage `json:"config"` + TransferMode string `json:"transferMode"` } // GetTaskSpec 拉取任务规格 @@ -149,6 +151,7 @@ type RecordUpdate struct { Checksum string `json:"checksum,omitempty"` StoragePath string `json:"storagePath,omitempty"` StorageTargetID uint `json:"storageTargetId,omitempty"` + StorageTransferMode string `json:"storageTransferMode,omitempty"` StorageUploadResults []StorageResultItem `json:"storageUploadResults,omitempty"` ErrorMessage string `json:"errorMessage,omitempty"` LogAppend string `json:"logAppend,omitempty"` @@ -160,6 +163,7 @@ type StorageResultItem struct { Status string `json:"status"` StoragePath string `json:"storagePath,omitempty"` FileSize int64 `json:"fileSize,omitempty"` + TransferMode string `json:"transferMode,omitempty"` Error string `json:"error,omitempty"` } @@ -169,6 +173,39 @@ func (c *MasterClient) UpdateRecord(ctx context.Context, recordID uint, update R return c.do(ctx, http.MethodPost, path, update, nil) } +// UploadArtifact streams an artifact through the Master for storage targets +// that are not directly reachable from the Agent. +func (c *MasterClient) UploadArtifact(ctx context.Context, recordID, targetID uint, objectKey string, size int64, checksum string, reader io.Reader) error { + path := fmt.Sprintf("/api/agent/records/%d/artifacts/%d", recordID, targetID) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.baseURL+path, reader) + if err != nil { + return err + } + // The executor owns and closes the artifact file. Prevent net/http from + // closing that underlying reader when it finishes the request body. + req.Body = io.NopCloser(reader) + req.ContentLength = size + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("X-Agent-Token", c.token) + req.Header.Set("X-BackupX-Object-Key", objectKey) + req.Header.Set("X-BackupX-SHA256", checksum) + client := *c.httpClient + client.Timeout = 0 + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("relay artifact to Master: %w", err) + } + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + closeErr := resp.Body.Close() + if readErr != nil || closeErr != nil { + return errors.Join(readErr, closeErr) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("relay artifact to Master: http %d: %s", resp.StatusCode, string(data)) + } + return nil +} + // RestoreSpec 与 service.AgentRestoreSpec 对齐 type RestoreSpec struct { RestoreRecordID uint `json:"restoreRecordId"` @@ -210,6 +247,27 @@ func (c *MasterClient) GetRestoreSpec(ctx context.Context, restoreRecordID uint) return &spec, nil } +func (c *MasterClient) DownloadRestoreArtifact(ctx context.Context, restoreRecordID uint) (io.ReadCloser, error) { + path := fmt.Sprintf("/api/agent/restores/%d/artifact", restoreRecordID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Agent-Token", c.token) + client := *c.httpClient + client.Timeout = 0 + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("download relayed artifact from Master: %w", err) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp.Body, nil + } + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + closeErr := resp.Body.Close() + return nil, errors.Join(fmt.Errorf("download relayed artifact from Master: http %d: %s", resp.StatusCode, string(data)), readErr, closeErr) +} + // UpdateRestore 上报恢复记录的状态/日志 func (c *MasterClient) UpdateRestore(ctx context.Context, restoreRecordID uint, update RestoreUpdate) error { path := fmt.Sprintf("/api/agent/restores/%d", restoreRecordID) diff --git a/server/internal/agent/executor.go b/server/internal/agent/executor.go index b1163b6..e58ac7d 100644 --- a/server/internal/agent/executor.go +++ b/server/internal/agent/executor.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -123,7 +124,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er } fileName := filepath.Base(finalPath) fileSize := info.Size() - storagePath := backup.BuildStorageKey(spec.Type, startedAt, fileName) + storagePath := backup.BuildRecordStorageKey(spec.Type, startedAt, recordID, fileName) // 5) 计算 checksum（一次读一次）并上传到所有目标 checksum, err := computeFileSHA256(finalPath) @@ -137,13 +138,15 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er } uploadResults := make([]StorageResultItem, 0, len(spec.StorageTargets)) selectedStorageTargetID := uint(0) + selectedStorageTransferMode := "" var uploadErrors []string for _, target := range spec.StorageTargets { - if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, spec.TaskID); err != nil { + if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, checksum, spec.TaskID); err != nil { uploadResults = append(uploadResults, StorageResultItem{ StorageTargetID: target.ID, StorageTargetName: target.Name, Status: "failed", + TransferMode: target.TransferMode, Error: err.Error(), }) uploadErrors = append(uploadErrors, fmt.Sprintf("%s: %v", target.Name, err)) @@ -152,6 +155,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er } if selectedStorageTargetID == 0 { selectedStorageTargetID = target.ID + selectedStorageTransferMode = target.TransferMode } uploadResults = append(uploadResults, StorageResultItem{ StorageTargetID: target.ID, @@ -159,6 +163,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er Status: "success", StoragePath: storagePath, FileSize: fileSize, + TransferMode: target.TransferMode, }) e.appendLog(ctx, recordID, fmt.Sprintf("[agent] 已上传到存储目标 %s\n", target.Name)) } @@ -179,34 +184,40 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er Checksum: checksum, StoragePath: storagePath, StorageTargetID: selectedStorageTargetID, + StorageTransferMode: selectedStorageTransferMode, StorageUploadResults: uploadResults, LogAppend: fmt.Sprintf("[agent] 任务完成，总计 %d 字节\n", fileSize), }) } // uploadToTarget 上传单个目标。为保持简化不做上传级重试（rclone 本身已有 low-level 重试）。 -func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, taskID uint) error { +func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, checksum string, taskID uint) error { + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("open artifact: %w", err) + } + if target.TransferMode == storage.TransferModeMasterRelay { + uploadErr := e.client.UploadArtifact(ctx, recordID, target.ID, objectKey, fileSize, checksum, f) + return errors.Join(uploadErr, f.Close()) + } var rawConfig map[string]any if len(target.Config) > 0 { // DecodeRawConfig 通过 json 解析 if err := jsonUnmarshalMap(target.Config, &rawConfig); err != nil { - return fmt.Errorf("parse storage config: %w", err) + return errors.Join(fmt.Errorf("parse storage config: %w", err), f.Close()) } } provider, err := e.storageRegistry.Create(ctx, target.Type, rawConfig) if err != nil { - return fmt.Errorf("create provider: %w", err) + closeErr := f.Close() + return errors.Join(fmt.Errorf("create provider: %w", err), closeErr) } - f, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("open artifact: %w", err) - } - defer f.Close() meta := map[string]string{ "taskId": fmt.Sprintf("%d", taskID), "recordId": fmt.Sprintf("%d", recordID), } - return provider.Upload(ctx, objectKey, f, fileSize, meta) + uploadErr := provider.Upload(ctx, objectKey, f, fileSize, meta) + return errors.Join(uploadErr, f.Close()) } // appendLog 追加日志到 Master 记录（尽力而为，失败不中断主流程） @@ -328,7 +339,7 @@ func (e *Executor) DeleteStorageObject(ctx context.Context, targetType string, t // ExecuteRestore 处理 restore_record 命令：拉规格 → 下载 → 解压 → 执行 runner.Restore → 上报结果。 // // 与 ExecuteRunTask 对称，但方向相反： -// - 下载：通过 spec.Storage 创建 provider → Download(spec.StoragePath) +// - 下载：直连共享存储，或通过 Master 中转其本地磁盘对象 // - 解密：当前 Agent 不支持加密恢复（密钥未下发），spec.Encrypt=true 会直接失败 // - 执行：backup.Registry.Runner(spec.Type).Restore // - 上报：通过 UpdateRestore（status/logAppend） @@ -357,28 +368,31 @@ func (e *Executor) ExecuteRestore(ctx context.Context, restoreRecordID uint) err } defer os.RemoveAll(tmpDir) - // 1) 创建 storage provider - var rawConfig map[string]any - if len(spec.Storage.Config) > 0 { - if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil { - e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err)) - return err - } - } - provider, err := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig) - if err != nil { - e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", err)) - return err - } - - // 2) 下载 + // 1) 下载 fileName := spec.FileName if strings.TrimSpace(fileName) == "" { fileName = filepath.Base(spec.StoragePath) } artifactPath := filepath.Join(tmpDir, filepath.Base(fileName)) e.appendRestoreLog(ctx, restoreRecordID, fmt.Sprintf("[agent] 下载备份文件 %s\n", spec.StoragePath)) - reader, err := provider.Download(ctx, spec.StoragePath) + var reader io.ReadCloser + if spec.Storage.TransferMode == storage.TransferModeMasterRelay { + reader, err = e.client.DownloadRestoreArtifact(ctx, restoreRecordID) + } else { + var rawConfig map[string]any + if len(spec.Storage.Config) > 0 { + if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil { + e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err)) + return err + } + } + provider, providerErr := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig) + if providerErr != nil { + e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", providerErr)) + return providerErr + } + reader, err = provider.Download(ctx, spec.StoragePath) + } if err != nil { e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("下载备份失败: %v", err)) return err @@ -489,8 +503,10 @@ func buildRestoreBackupTaskSpec(spec *RestoreSpec, startedAt time.Time, tempDir } // writeReaderToLocal 把 reader 写到本地文件（Agent 侧工具函数）。 -func writeReaderToLocal(targetPath string, reader io.ReadCloser) error { - defer reader.Close() +func writeReaderToLocal(targetPath string, reader io.ReadCloser) (err error) { + defer func() { + err = errors.Join(err, reader.Close()) + }() if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return err } @@ -498,9 +514,8 @@ func writeReaderToLocal(targetPath string, reader io.ReadCloser) error { if err != nil { return err } - defer file.Close() - _, err = io.Copy(file, reader) - return err + _, copyErr := io.Copy(file, reader) + return errors.Join(copyErr, file.Close()) } // 辅助函数 diff --git a/server/internal/agent/executor_test.go b/server/internal/agent/executor_test.go index d4f35ff..b197418 100644 --- a/server/internal/agent/executor_test.go +++ b/server/internal/agent/executor_test.go @@ -1,7 +1,10 @@ package agent import ( + "archive/tar" + "bytes" "context" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -109,6 +112,150 @@ func TestExecuteRunTaskRecordsPerTargetUploadResults(t *testing.T) { } } +func TestExecuteRunTaskRelaysMasterLocalDiskTarget(t *testing.T) { + sourceDir := t.TempDir() + if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("centralize me"), 0o644); err != nil { + t.Fatalf("WriteFile returned error: %v", err) + } + var relayed []byte + var finalUpdate RecordUpdate + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/agent/tasks/1": + writeAgentEnvelope(t, w, TaskSpec{ + TaskID: 1, + Name: "remote-source", + Type: "file", + SourcePath: sourceDir, + Compression: "gzip", + StorageTargets: []StorageTargetConfig{{ + ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay, + }}, + }) + case r.Method == http.MethodPut && r.URL.Path == "/api/agent/records/99/artifacts/11": + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("ReadAll relayed body: %v", err) + } + digest := sha256.Sum256(body) + if got := r.Header.Get("X-BackupX-SHA256"); got != fmt.Sprintf("%x", digest[:]) { + t.Fatalf("relay checksum header = %q", got) + } + objectKey := r.Header.Get("X-BackupX-Object-Key") + if !strings.Contains(objectKey, "/records/99/") || r.ContentLength != int64(len(body)) { + t.Fatalf("invalid relay metadata: key=%q length=%d body=%d", objectKey, r.ContentLength, len(body)) + } + relayed = append([]byte(nil), body...) + writeAgentEnvelope(t, w, map[string]string{"status": "ok"}) + case r.Method == http.MethodPost && r.URL.Path == "/api/agent/records/99": + var update RecordUpdate + if err := json.NewDecoder(r.Body).Decode(&update); err != nil { + t.Fatalf("Decode update returned error: %v", err) + } + if update.Status != "" { + finalUpdate = update + } + writeAgentEnvelope(t, w, map[string]string{"status": "ok"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp")) + if err := executor.ExecuteRunTask(context.Background(), 1, 99); err != nil { + t.Fatalf("ExecuteRunTask returned error: %v", err) + } + if len(relayed) == 0 { + t.Fatal("expected artifact bytes to be streamed through Master") + } + if finalUpdate.Status != "success" || finalUpdate.StorageTransferMode != storage.TransferModeMasterRelay { + t.Fatalf("unexpected final relay update: %#v", finalUpdate) + } + if len(finalUpdate.StorageUploadResults) != 1 || finalUpdate.StorageUploadResults[0].TransferMode != storage.TransferModeMasterRelay { + t.Fatalf("unexpected relay target result: %#v", finalUpdate.StorageUploadResults) + } +} + +func TestExecuteRestoreDownloadsMasterRelayedArtifact(t *testing.T) { + var archive bytes.Buffer + tarWriter := tar.NewWriter(&archive) + content := []byte("restored through Master") + header := &tar.Header{Name: "site/index.html", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg} + if err := tarWriter.WriteHeader(header); err != nil { + t.Fatalf("WriteHeader returned error: %v", err) + } + if _, err := tarWriter.Write(content); err != nil { + t.Fatalf("Write returned error: %v", err) + } + if err := tarWriter.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + artifact := append([]byte(nil), archive.Bytes()...) + digest := sha256.Sum256(artifact) + restoreRoot := t.TempDir() + restoreSource := filepath.Join(restoreRoot, "site") + artifactRequests := 0 + var finalUpdate RestoreUpdate + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/spec": + writeAgentEnvelope(t, w, RestoreSpec{ + RestoreRecordID: 77, + BackupRecordID: 99, + TaskID: 1, + TaskName: "remote-source", + Type: "file", + SourcePath: restoreSource, + Storage: StorageTargetConfig{ + ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay, + }, + StoragePath: "BackupX/file/site.tar", + FileName: "site.tar", + Checksum: fmt.Sprintf("%x", digest[:]), + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/artifact": + artifactRequests++ + if r.Header.Get("X-Agent-Token") != "token" { + t.Fatalf("missing Agent token on relay download") + } + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(artifact))) + _, _ = w.Write(artifact) + case r.Method == http.MethodPost && r.URL.Path == "/api/agent/restores/77": + var update RestoreUpdate + if err := json.NewDecoder(r.Body).Decode(&update); err != nil { + t.Fatalf("Decode update returned error: %v", err) + } + if update.Status != "" { + finalUpdate = update + } + writeAgentEnvelope(t, w, map[string]string{"status": "ok"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp")) + if err := executor.ExecuteRestore(context.Background(), 77); err != nil { + t.Fatalf("ExecuteRestore returned error: %v", err) + } + if artifactRequests != 1 { + t.Fatalf("expected one relay artifact request, got %d", artifactRequests) + } + restored, err := os.ReadFile(filepath.Join(restoreRoot, "site", "index.html")) + if err != nil { + t.Fatalf("ReadFile returned error: %v", err) + } + if string(restored) != string(content) { + t.Fatalf("restored content = %q, want %q", restored, content) + } + if finalUpdate.Status != "success" { + t.Fatalf("unexpected final restore update: %#v", finalUpdate) + } +} + func TestExecuteRunTaskReportsPerTargetUploadResultsWhenAllTargetsFail(t *testing.T) { sourceDir := t.TempDir() if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("hello"), 0o644); err != nil { diff --git a/server/internal/app/app.go b/server/internal/app/app.go index 88deb5f..2cfac10 100644 --- a/server/internal/app/app.go +++ b/server/internal/app/app.go @@ -84,7 +84,7 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application, // nodeRepo 在下方 Cluster 节点管理区块才实例化，这里延后注入 backupRunnerRegistry := backup.NewRegistry(backup.NewFileRunner(), backup.NewSQLiteRunner(), backup.NewMySQLRunner(nil), backup.NewPostgreSQLRunner(nil), backup.NewSAPHANARunner(nil), backup.NewMongoDBRunner(nil)) logHub := backup.NewLogHub() - retentionService := backupretention.NewService(backupRecordRepo) + retentionService := backupretention.NewService(backupRecordRepo, configCipher.Key()) notifyRegistry := notify.NewRegistry(notify.NewEmailNotifier(), notify.NewWebhookNotifier(), notify.NewTelegramNotifier()) notificationService := service.NewNotificationService(notificationRepo, notifyRegistry, configCipher) authService.SetNotificationService(notificationService) @@ -135,7 +135,7 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application, // Agent 协议服务：命令队列 + 任务下发 + 记录上报 agentCmdRepo := repository.NewAgentCommandRepository(db) nodeService.SetAgentCommandRepository(agentCmdRepo) - agentService := service.NewAgentService(nodeRepo, backupTaskRepo, backupRecordRepo, storageTargetRepo, agentCmdRepo, configCipher) + agentService := service.NewAgentService(nodeRepo, backupTaskRepo, backupRecordRepo, storageTargetRepo, agentCmdRepo, configCipher, storageRegistry) agentService.SetRestoreRepository(restoreRecordRepo) agentService.StartCommandTimeoutMonitor(ctx, 30*time.Second, 10*time.Minute) diff --git a/server/internal/backup/repository.go b/server/internal/backup/repository.go new file mode 100644 index 0000000..da0659c --- /dev/null +++ b/server/internal/backup/repository.go @@ -0,0 +1,1356 @@ +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "backupx/server/internal/storage" + "github.com/klauspost/compress/zstd" +) + +const ( + repositoryFormatVersion = 1 + repositoryRoot = ".backupx/repository/v1" + repositoryIndexPrefix = repositoryRoot + "/indexes" + repositoryPackPrefix = repositoryRoot + "/packs" + repositorySnapshotRoot = repositoryRoot + "/snapshots" + repositoryDefaultPack = int64(32 << 20) + repositoryMaxIndexSize = int64(64 << 20) + repositoryMaxSnapshot = int64(256 << 20) + repositoryMaxEncoded = int64(repositoryChunkMax + (256 << 10)) +) + +type RepositoryStore struct { + key []byte + chunker *contentDefinedChunker + packSize int64 +} + +type RepositoryPlan struct { + tempDir string + spoolPath string + chunks map[string]repositoryPlanChunk + chunkOrder []string + snapshot repositorySnapshot + Manifest Manifest + LogicalSize int64 + UniqueSize int64 +} + +type repositoryPlanChunk struct { + Offset int64 + Size int64 +} + +type repositorySnapshot struct { + Version int `json:"version"` + TaskID uint `json:"taskId"` + CreatedAt time.Time `json:"createdAt"` + Compression string `json:"compression"` + Encrypted bool `json:"encrypted"` + SourcePaths []string `json:"sourcePaths"` + Entries []repositoryEntry `json:"entries"` +} + +type repositoryEntry struct { + Path string `json:"path"` + Kind string `json:"kind"` + Mode uint32 `json:"mode"` + ModTime int64 `json:"modTime"` + Size int64 `json:"size"` + LinkTarget string `json:"linkTarget,omitempty"` + Chunks []string `json:"chunks,omitempty"` +} + +type repositorySnapshotEnvelope struct { + Version int `json:"version"` + Encrypted bool `json:"encrypted"` + Data json.RawMessage `json:"data,omitempty"` + Ciphertext string `json:"ciphertext,omitempty"` +} + +type repositoryChunkLocation struct { + Pack string `json:"pack"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` + PlainSize int64 `json:"plainSize"` + Compression string `json:"compression"` + Encrypted bool `json:"encrypted"` +} + +type repositoryIndexSegment struct { + Version int `json:"version"` + CreatedAt time.Time `json:"createdAt"` + Pack string `json:"pack"` + Chunks map[string]repositoryChunkLocation `json:"chunks"` +} + +type RepositoryUploadResult struct { + SnapshotKey string + SnapshotSize int64 + LogicalSize int64 + UploadedBytes int64 + ReusedBytes int64 + UniqueChunks int + NewChunks int + Checksum string +} + +type RepositoryVerifyResult struct { + Entries int + Chunks int + Bytes int64 +} + +type RepositoryPruneResult struct { + DeletedIndexes int + DeletedPacks int + ReclaimedBytes int64 +} + +func NewRepositoryStore(encryptionKey []byte) *RepositoryStore { + keyCopy := make([]byte, len(encryptionKey)) + copy(keyCopy, encryptionKey) + return &RepositoryStore{ + key: keyCopy, + chunker: newContentDefinedChunker(), + packSize: repositoryDefaultPack, + } +} + +func (s *RepositoryStore) SnapshotKey(taskID, recordID uint, startedAt time.Time) string { + stamp := startedAt.UTC().Format("20060102T150405.000000000Z") + return fmt.Sprintf("%s/%d/%s-%d.bxrs", repositorySnapshotRoot, taskID, stamp, recordID) +} + +func (p *RepositoryPlan) Close() error { + if p == nil || strings.TrimSpace(p.tempDir) == "" { + return nil + } + err := os.RemoveAll(p.tempDir) + p.tempDir = "" + p.spoolPath = "" + return err +} + +func (s *RepositoryStore) BuildPlan(ctx context.Context, task TaskSpec, writer LogWriter) (plan *RepositoryPlan, err error) { + if writer == nil { + writer = NopLogWriter{} + } + sourcePaths := compactPaths(task.SourcePaths) + if len(sourcePaths) == 0 && strings.TrimSpace(task.SourcePath) != "" { + sourcePaths = []string{filepath.Clean(strings.TrimSpace(task.SourcePath))} + } + if len(sourcePaths) == 0 { + return nil, fmt.Errorf("source path is required") + } + compression, err := s.normalizeCompression(task.Compression) + if err != nil { + return nil, err + } + if task.Encrypt && len(s.key) != 32 { + return nil, fmt.Errorf("repository encryption requires a 256-bit key") + } + if err := os.MkdirAll(task.TempDir, 0o755); err != nil { + return nil, fmt.Errorf("create repository temp root: %w", err) + } + tempDir, err := os.MkdirTemp(task.TempDir, "repository-plan-*") + if err != nil { + return nil, fmt.Errorf("create repository plan directory: %w", err) + } + defer func() { + if err == nil { + return + } + if cleanupErr := os.RemoveAll(tempDir); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("clean repository plan: %w", cleanupErr)) + } + }() + + spoolPath := filepath.Join(tempDir, "chunks.spool") + spool, err := os.Create(spoolPath) + if err != nil { + return nil, fmt.Errorf("create repository chunk spool: %w", err) + } + spoolClosed := false + defer func() { + if spoolClosed { + return + } + if closeErr := spool.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("close repository chunk spool: %w", closeErr)) + } + }() + + plan = &RepositoryPlan{ + tempDir: tempDir, + spoolPath: spoolPath, + chunks: make(map[string]repositoryPlanChunk), + snapshot: repositorySnapshot{ + Version: repositoryFormatVersion, + TaskID: task.ID, + CreatedAt: task.StartedAt.UTC(), + Compression: compression, + Encrypted: task.Encrypt, + SourcePaths: sourcePaths, + Entries: make([]repositoryEntry, 0), + }, + Manifest: Manifest{Entries: make([]ManifestEntry, 0)}, + } + excludes := normalizeExcludePatterns(task.ExcludePatterns) + seenPaths := make(map[string]struct{}) + spoolOffset := int64(0) + writer.WriteLine(fmt.Sprintf("CDC 仓库模式：FastCDC %d KiB/%d KiB/%d KiB，pack %d MiB", repositoryChunkMin>>10, repositoryChunkAvg>>10, repositoryChunkMax>>10, s.packSize>>20)) + + for sourceIndex, rawSource := range sourcePaths { + sourcePath := filepath.Clean(strings.TrimSpace(rawSource)) + if _, statErr := os.Lstat(sourcePath); statErr != nil { + return nil, fmt.Errorf("stat source path %s: %w", sourcePath, statErr) + } + baseParent := filepath.Dir(sourcePath) + writer.WriteLine(fmt.Sprintf("扫描源路径 [%d/%d]：%s", sourceIndex+1, len(sourcePaths), sourcePath)) + walkErr := filepath.Walk(sourcePath, func(currentPath string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + relative, relErr := filepath.Rel(baseParent, currentPath) + if relErr != nil { + return relErr + } + archiveName := path.Clean(filepath.ToSlash(relative)) + if archiveName == "." || strings.HasPrefix(archiveName, "../") { + return fmt.Errorf("invalid repository entry path %q", archiveName) + } + if shouldExcludeEntry(archiveName, info.IsDir(), excludes) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if _, duplicated := seenPaths[archiveName]; duplicated { + return fmt.Errorf("duplicate repository entry %q from overlapping source paths", archiveName) + } + seenPaths[archiveName] = struct{}{} + + entry := repositoryEntry{ + Path: archiveName, + Mode: uint32(info.Mode().Perm()), + ModTime: info.ModTime().UTC().UnixNano(), + Size: info.Size(), + } + manifestEntry := entryFromInfo(archiveName, info) + switch { + case info.IsDir(): + entry.Kind = "directory" + case info.Mode()&os.ModeSymlink != 0: + entry.Kind = "symlink" + linkTarget, linkErr := os.Readlink(currentPath) + if linkErr != nil { + return fmt.Errorf("read symlink %s: %w", currentPath, linkErr) + } + entry.LinkTarget = linkTarget + case info.Mode().IsRegular(): + entry.Kind = "file" + plan.LogicalSize += info.Size() + file, openErr := os.Open(currentPath) + if openErr != nil { + return fmt.Errorf("open source file %s: %w", currentPath, openErr) + } + splitErr := s.chunker.Split(ctx, file, func(raw []byte) error { + chunkID := s.chunkID(raw, compression, task.Encrypt) + entry.Chunks = append(entry.Chunks, chunkID) + if _, exists := plan.chunks[chunkID]; exists { + return nil + } + written, writeErr := spool.Write(raw) + if writeErr != nil { + return fmt.Errorf("write repository chunk spool: %w", writeErr) + } + if written != len(raw) { + return io.ErrShortWrite + } + plan.chunks[chunkID] = repositoryPlanChunk{Offset: spoolOffset, Size: int64(len(raw))} + plan.chunkOrder = append(plan.chunkOrder, chunkID) + plan.UniqueSize += int64(len(raw)) + spoolOffset += int64(len(raw)) + return nil + }) + closeErr := file.Close() + if splitErr != nil { + return errors.Join(fmt.Errorf("chunk source file %s: %w", currentPath, splitErr), closeErr) + } + if closeErr != nil { + return fmt.Errorf("close source file %s: %w", currentPath, closeErr) + } + default: + writer.WriteLine(fmt.Sprintf("跳过不支持的特殊文件：%s", currentPath)) + delete(seenPaths, archiveName) + return nil + } + plan.snapshot.Entries = append(plan.snapshot.Entries, entry) + plan.Manifest.Entries = append(plan.Manifest.Entries, manifestEntry) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("scan source path %s: %w", sourcePath, walkErr) + } + } + if err := spool.Sync(); err != nil { + return nil, fmt.Errorf("sync repository chunk spool: %w", err) + } + if err := spool.Close(); err != nil { + return nil, fmt.Errorf("close repository chunk spool: %w", err) + } + spoolClosed = true + if err := s.validateSnapshot(&plan.snapshot); err != nil { + return nil, err + } + writer.WriteLine(fmt.Sprintf("CDC 扫描完成：%d 个条目，逻辑数据 %d bytes，任务内唯一数据 %d bytes", len(plan.snapshot.Entries), plan.LogicalSize, plan.UniqueSize)) + return plan, nil +} + +func (s *RepositoryStore) Upload(ctx context.Context, provider storage.StorageProvider, plan *RepositoryPlan, snapshotKey string) (*RepositoryUploadResult, error) { + if provider == nil || plan == nil || strings.TrimSpace(snapshotKey) == "" { + return nil, fmt.Errorf("repository provider, plan and snapshot key are required") + } + locations, err := s.loadIndex(ctx, provider) + if err != nil { + return nil, err + } + spool, err := os.Open(plan.spoolPath) + if err != nil { + return nil, fmt.Errorf("open repository chunk spool: %w", err) + } + uploadedBytes, newChunks, err := s.uploadMissingChunks(ctx, provider, plan, spool, locations) + closeErr := spool.Close() + if err != nil { + return nil, errors.Join(err, closeErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close repository chunk spool: %w", closeErr) + } + + snapshotBytes, err := s.encodeSnapshot(plan.snapshot) + if err != nil { + return nil, err + } + if err := provider.Upload(ctx, snapshotKey, bytes.NewReader(snapshotBytes), int64(len(snapshotBytes)), map[string]string{"format": "backupx-repository-v1"}); err != nil { + return nil, fmt.Errorf("upload repository snapshot %s: %w", snapshotKey, err) + } + digest := sha256.Sum256(snapshotBytes) + reusedBytes := int64(0) + for _, chunkID := range plan.chunkOrder { + if _, wasNew := newChunks[chunkID]; !wasNew { + reusedBytes += plan.chunks[chunkID].Size + } + } + return &RepositoryUploadResult{ + SnapshotKey: snapshotKey, + SnapshotSize: int64(len(snapshotBytes)), + LogicalSize: plan.LogicalSize, + UploadedBytes: uploadedBytes + int64(len(snapshotBytes)), + ReusedBytes: reusedBytes, + UniqueChunks: len(plan.chunkOrder), + NewChunks: len(newChunks), + Checksum: hex.EncodeToString(digest[:]), + }, nil +} + +// EstimateUploadSize returns a target-aware soft-quota estimate. Existing +// chunks are excluded so a nearly full repository can still accept a snapshot +// that is almost entirely deduplicated. +func (s *RepositoryStore) EstimateUploadSize(ctx context.Context, provider storage.StorageProvider, plan *RepositoryPlan) (int64, error) { + if provider == nil || plan == nil { + return 0, fmt.Errorf("repository provider and plan are required") + } + locations, err := s.loadIndex(ctx, provider) + if err != nil { + return 0, err + } + snapshotBytes, err := s.encodeSnapshot(plan.snapshot) + if err != nil { + return 0, err + } + estimate := int64(len(snapshotBytes)) + for _, chunkID := range plan.chunkOrder { + if _, exists := locations[chunkID]; exists { + continue + } + // Compression generally lowers this value. The small allowance covers + // encryption tags and index metadata while keeping the check conservative. + estimate += plan.chunks[chunkID].Size + 256 + } + return estimate, nil +} + +func (s *RepositoryStore) Restore(ctx context.Context, provider storage.StorageProvider, snapshotKey, expectedChecksum string, task TaskSpec, writer LogWriter) (err error) { + if writer == nil { + writer = NopLogWriter{} + } + if strings.TrimSpace(expectedChecksum) == "" { + return fmt.Errorf("repository snapshot checksum is required for restore") + } + snapshot, _, err := s.loadSnapshot(ctx, provider, snapshotKey, expectedChecksum) + if err != nil { + return err + } + locations, err := s.loadIndex(ctx, provider) + if err != nil { + return err + } + restoreSource := strings.TrimSpace(task.SourcePath) + if len(task.SourcePaths) > 0 { + restoreSource = strings.TrimSpace(task.SourcePaths[0]) + } + targetRoot := strings.TrimSpace(task.RestoreTargetPath) + if targetRoot == "" { + if restoreSource == "" { + return fmt.Errorf("repository restore source path is required when no restore target is provided") + } + targetRoot = filepath.Dir(filepath.Clean(restoreSource)) + } else { + targetRoot = filepath.Clean(targetRoot) + } + if !filepath.IsAbs(targetRoot) { + return fmt.Errorf("repository restore target must be absolute: %s", targetRoot) + } + restoreRoot, err := s.openRestoreRoot(targetRoot) + if err != nil { + return fmt.Errorf("open repository restore root: %w", err) + } + defer func() { + err = errors.Join(err, restoreRoot.Close()) + }() + + restored := 0 + directories := make([]repositoryEntry, 0) + for _, entry := range snapshot.Entries { + if err := ctx.Err(); err != nil { + return err + } + if len(task.SelectedPaths) > 0 && !pathSelected(entry.Path, task.SelectedPaths) { + continue + } + entryPath, localizeErr := filepath.Localize(entry.Path) + if localizeErr != nil || !filepath.IsLocal(entryPath) { + return fmt.Errorf("unsafe repository restore path %q", entry.Path) + } + if parent := filepath.Dir(entryPath); parent != "." { + if err := restoreRoot.MkdirAll(parent, 0o755); err != nil { + return fmt.Errorf("create restore parent for %s: %w", entry.Path, err) + } + } + switch entry.Kind { + case "directory": + if info, statErr := restoreRoot.Lstat(entryPath); statErr == nil { + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("restore directory crosses existing symlink: %s", entry.Path) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("inspect restore directory %s: %w", entry.Path, statErr) + } + if err := restoreRoot.MkdirAll(entryPath, os.FileMode(entry.Mode)); err != nil { + return fmt.Errorf("create restore directory %s: %w", entry.Path, err) + } + directories = append(directories, entry) + case "symlink": + resolvedLinkTarget := path.Clean(path.Join(path.Dir(entry.Path), strings.ReplaceAll(entry.LinkTarget, "\\", "/"))) + localizedLinkTarget, localizeTargetErr := filepath.Localize(resolvedLinkTarget) + if localizeTargetErr != nil || !filepath.IsLocal(localizedLinkTarget) { + return fmt.Errorf("unsafe repository symlink target %q", entry.LinkTarget) + } + linkTarget, relativeTargetErr := filepath.Rel(filepath.Dir(entryPath), localizedLinkTarget) + if relativeTargetErr != nil { + return fmt.Errorf("resolve restore symlink target %s: %w", entry.Path, relativeTargetErr) + } + if info, statErr := restoreRoot.Lstat(entryPath); statErr == nil { + if info.IsDir() { + return fmt.Errorf("refuse to replace restore directory with symlink: %s", entry.Path) + } + if err := restoreRoot.Remove(entryPath); err != nil { + return fmt.Errorf("replace restore symlink %s: %w", entry.Path, err) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("inspect restore symlink %s: %w", entry.Path, statErr) + } + if err := restoreRoot.Symlink(linkTarget, entryPath); err != nil { + return fmt.Errorf("create restore symlink %s: %w", entry.Path, err) + } + case "file": + if info, statErr := restoreRoot.Lstat(entryPath); statErr == nil { + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("restore file would overwrite existing symlink: %s", entry.Path) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("inspect restore file %s: %w", entry.Path, statErr) + } + file, openErr := restoreRoot.OpenFile(entryPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(entry.Mode)) + if openErr != nil { + return fmt.Errorf("create restore file %s: %w", entry.Path, openErr) + } + var written int64 + for _, chunkID := range entry.Chunks { + raw, readErr := s.readChunk(ctx, provider, chunkID, locations) + if readErr != nil { + return errors.Join(fmt.Errorf("restore %s: %w", entry.Path, readErr), file.Close()) + } + count, writeErr := file.Write(raw) + written += int64(count) + if writeErr != nil { + return errors.Join(fmt.Errorf("write restore file %s: %w", entry.Path, writeErr), file.Close()) + } + if count != len(raw) { + return errors.Join(io.ErrShortWrite, file.Close()) + } + } + if written != entry.Size { + return errors.Join(fmt.Errorf("restored size mismatch for %s: expected %d, got %d", entry.Path, entry.Size, written), file.Close()) + } + if err := file.Chmod(os.FileMode(entry.Mode)); err != nil { + return errors.Join(fmt.Errorf("restore mode for %s: %w", entry.Path, err), file.Close()) + } + if closeErr := file.Close(); closeErr != nil { + return fmt.Errorf("close restore file %s: %w", entry.Path, closeErr) + } + modTime := time.Unix(0, entry.ModTime) + if err := restoreRoot.Chtimes(entryPath, modTime, modTime); err != nil { + return fmt.Errorf("restore timestamp for %s: %w", entry.Path, err) + } + default: + return fmt.Errorf("unsupported repository entry kind %q", entry.Kind) + } + restored++ + } + // Children modify their parent directory timestamps, so directory metadata + // is restored only after every selected file has been written. + for index := len(directories) - 1; index >= 0; index-- { + entry := directories[index] + entryPath, localizeErr := filepath.Localize(entry.Path) + if localizeErr != nil || !filepath.IsLocal(entryPath) { + return fmt.Errorf("unsafe repository restore path %q", entry.Path) + } + if err := restoreRoot.Chmod(entryPath, os.FileMode(entry.Mode)); err != nil { + return fmt.Errorf("restore mode for %s: %w", entry.Path, err) + } + modTime := time.Unix(0, entry.ModTime) + if err := restoreRoot.Chtimes(entryPath, modTime, modTime); err != nil { + return fmt.Errorf("restore timestamp for %s: %w", entry.Path, err) + } + } + writer.WriteLine(fmt.Sprintf("CDC 仓库恢复完成：%d 个条目", restored)) + return nil +} + +func (s *RepositoryStore) ExportTar(ctx context.Context, provider storage.StorageProvider, snapshotKey, expectedChecksum, destination string) (err error) { + if strings.TrimSpace(expectedChecksum) == "" { + return fmt.Errorf("repository snapshot checksum is required for export") + } + snapshot, _, err := s.loadSnapshot(ctx, provider, snapshotKey, expectedChecksum) + if err != nil { + return err + } + locations, err := s.loadIndex(ctx, provider) + if err != nil { + return err + } + file, err := os.Create(destination) + if err != nil { + return fmt.Errorf("create repository export: %w", err) + } + tw := tar.NewWriter(file) + tarClosed := false + fileClosed := false + defer func() { + if !tarClosed { + err = errors.Join(err, tw.Close()) + } + if !fileClosed { + err = errors.Join(err, file.Close()) + } + }() + for _, entry := range snapshot.Entries { + if err := ctx.Err(); err != nil { + return err + } + header := &tar.Header{ + Name: entry.Path, + Mode: int64(entry.Mode), + ModTime: time.Unix(0, entry.ModTime), + Size: entry.Size, + } + switch entry.Kind { + case "directory": + header.Typeflag = tar.TypeDir + header.Size = 0 + header.Name = strings.TrimSuffix(entry.Path, "/") + "/" + case "symlink": + header.Typeflag = tar.TypeSymlink + header.Size = 0 + header.Linkname = entry.LinkTarget + case "file": + header.Typeflag = tar.TypeReg + default: + return fmt.Errorf("unsupported repository entry kind %q", entry.Kind) + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("write repository tar header: %w", err) + } + if entry.Kind == "file" { + for _, chunkID := range entry.Chunks { + raw, readErr := s.readChunk(ctx, provider, chunkID, locations) + if readErr != nil { + return readErr + } + if _, writeErr := tw.Write(raw); writeErr != nil { + return fmt.Errorf("write repository tar data: %w", writeErr) + } + } + } + } + closeErr := tw.Close() + tarClosed = true + if closeErr != nil { + return fmt.Errorf("close repository tar: %w", closeErr) + } + closeErr = file.Close() + fileClosed = true + if closeErr != nil { + return fmt.Errorf("close repository export: %w", closeErr) + } + return nil +} + +func (s *RepositoryStore) Verify(ctx context.Context, provider storage.StorageProvider, snapshotKey, expectedChecksum string) (*RepositoryVerifyResult, error) { + if strings.TrimSpace(expectedChecksum) == "" { + return nil, fmt.Errorf("repository snapshot checksum is required for verification") + } + snapshot, _, err := s.loadSnapshot(ctx, provider, snapshotKey, expectedChecksum) + if err != nil { + return nil, err + } + locations, err := s.loadIndex(ctx, provider) + if err != nil { + return nil, err + } + seen := make(map[string]struct{}) + result := &RepositoryVerifyResult{Entries: len(snapshot.Entries)} + for _, entry := range snapshot.Entries { + for _, chunkID := range entry.Chunks { + if _, checked := seen[chunkID]; checked { + continue + } + raw, readErr := s.readChunk(ctx, provider, chunkID, locations) + if readErr != nil { + return nil, readErr + } + seen[chunkID] = struct{}{} + result.Chunks++ + result.Bytes += int64(len(raw)) + } + } + return result, nil +} + +func (s *RepositoryStore) Prune(ctx context.Context, provider storage.StorageProvider) (*RepositoryPruneResult, error) { + if provider == nil { + return nil, fmt.Errorf("repository provider is required") + } + liveChunks := make(map[string]struct{}) + snapshots, err := provider.List(ctx, repositorySnapshotRoot) + if err != nil { + return nil, fmt.Errorf("list repository snapshots: %w", err) + } + sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Key < snapshots[j].Key }) + for _, object := range snapshots { + snapshot, _, loadErr := s.loadSnapshot(ctx, provider, object.Key, "") + if loadErr != nil { + return nil, fmt.Errorf("refuse to prune with unreadable snapshot %s: %w", object.Key, loadErr) + } + for _, entry := range snapshot.Entries { + for _, chunkID := range entry.Chunks { + liveChunks[chunkID] = struct{}{} + } + } + } + + indexObjects, err := provider.List(ctx, repositoryIndexPrefix) + if err != nil { + return nil, fmt.Errorf("list repository indexes: %w", err) + } + packSizes := make(map[string]int64) + packObjects, err := provider.List(ctx, repositoryPackPrefix) + if err != nil { + return nil, fmt.Errorf("list repository packs: %w", err) + } + for _, object := range packObjects { + packSizes[object.Key] = object.Size + } + keptPacks := make(map[string]struct{}) + result := &RepositoryPruneResult{} + for _, object := range indexObjects { + segment, readErr := s.readIndexSegment(ctx, provider, object.Key) + if readErr != nil { + return nil, fmt.Errorf("refuse to prune with unreadable index %s: %w", object.Key, readErr) + } + keep := false + for chunkID := range segment.Chunks { + if _, live := liveChunks[chunkID]; live { + keep = true + break + } + } + if keep { + keptPacks[segment.Pack] = struct{}{} + continue + } + if err := provider.Delete(ctx, object.Key); err != nil { + return nil, fmt.Errorf("delete unused repository index %s: %w", object.Key, err) + } + result.DeletedIndexes++ + if err := provider.Delete(ctx, segment.Pack); err != nil { + return nil, fmt.Errorf("delete unused repository pack %s: %w", segment.Pack, err) + } + result.DeletedPacks++ + result.ReclaimedBytes += packSizes[segment.Pack] + delete(packSizes, segment.Pack) + } + for packKey, size := range packSizes { + if _, keep := keptPacks[packKey]; keep { + continue + } + if err := provider.Delete(ctx, packKey); err != nil { + return nil, fmt.Errorf("delete orphaned repository pack %s: %w", packKey, err) + } + result.DeletedPacks++ + result.ReclaimedBytes += size + } + return result, nil +} + +func (s *RepositoryStore) uploadMissingChunks(ctx context.Context, provider storage.StorageProvider, plan *RepositoryPlan, spool *os.File, locations map[string]repositoryChunkLocation) (uploadedBytes int64, newChunkIDs map[string]struct{}, err error) { + newChunkIDs = make(map[string]struct{}) + packPath := "" + var packFile *os.File + packSize := int64(0) + packChunks := make(map[string]repositoryChunkLocation) + + closeAndRemove := func() error { + var cleanupErr error + if packFile != nil { + cleanupErr = packFile.Close() + packFile = nil + } + if packPath != "" { + cleanupErr = errors.Join(cleanupErr, os.Remove(packPath)) + packPath = "" + } + return cleanupErr + } + flush := func() error { + if packFile == nil || len(packChunks) == 0 { + return nil + } + if err := packFile.Sync(); err != nil { + return fmt.Errorf("sync repository pack: %w", err) + } + if err := packFile.Close(); err != nil { + return fmt.Errorf("close repository pack: %w", err) + } + packFile = nil + packKey, indexBytes, packBytes, err := s.uploadPack(ctx, provider, packPath, packChunks) + if err != nil { + return err + } + for chunkID, location := range packChunks { + location.Pack = packKey + locations[chunkID] = location + newChunkIDs[chunkID] = struct{}{} + } + uploadedBytes += packBytes + indexBytes + if err := os.Remove(packPath); err != nil { + return fmt.Errorf("remove temporary repository pack: %w", err) + } + packPath = "" + packSize = 0 + packChunks = make(map[string]repositoryChunkLocation) + return nil + } + defer func() { + if cleanupErr := closeAndRemove(); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("clean temporary repository pack: %w", cleanupErr)) + } + }() + + for _, chunkID := range plan.chunkOrder { + if _, exists := locations[chunkID]; exists { + continue + } + planChunk := plan.chunks[chunkID] + raw := make([]byte, planChunk.Size) + readCount, readErr := spool.ReadAt(raw, planChunk.Offset) + if readErr != nil && readErr != io.EOF { + return 0, nil, fmt.Errorf("read repository chunk spool: %w", readErr) + } + if int64(readCount) != planChunk.Size { + return 0, nil, io.ErrUnexpectedEOF + } + encoded, encodeErr := s.encodeChunk(raw, plan.snapshot.Compression, plan.snapshot.Encrypted, chunkID) + if encodeErr != nil { + return 0, nil, encodeErr + } + if packFile != nil && packSize > 0 && packSize+int64(len(encoded)) > s.packSize { + if err := flush(); err != nil { + return 0, nil, err + } + } + if packFile == nil { + created, createErr := os.CreateTemp(plan.tempDir, "repository-pack-*") + if createErr != nil { + return 0, nil, fmt.Errorf("create temporary repository pack: %w", createErr) + } + packFile = created + packPath = created.Name() + } + written, writeErr := packFile.Write(encoded) + if writeErr != nil { + return 0, nil, fmt.Errorf("write repository pack: %w", writeErr) + } + if written != len(encoded) { + return 0, nil, io.ErrShortWrite + } + packChunks[chunkID] = repositoryChunkLocation{ + Offset: packSize, + Length: int64(len(encoded)), + PlainSize: planChunk.Size, + Compression: plan.snapshot.Compression, + Encrypted: plan.snapshot.Encrypted, + } + packSize += int64(len(encoded)) + } + if err := flush(); err != nil { + return 0, nil, err + } + return uploadedBytes, newChunkIDs, nil +} + +func (s *RepositoryStore) uploadPack(ctx context.Context, provider storage.StorageProvider, packPath string, chunks map[string]repositoryChunkLocation) (packKey string, indexSize int64, packSize int64, err error) { + pack, err := os.Open(packPath) + if err != nil { + return "", 0, 0, fmt.Errorf("open repository pack: %w", err) + } + packClosed := false + defer func() { + if !packClosed { + err = errors.Join(err, pack.Close()) + } + }() + hasher := sha256.New() + packBytes, err := io.Copy(hasher, pack) + if err != nil { + return "", 0, 0, fmt.Errorf("hash repository pack: %w", err) + } + packID := hex.EncodeToString(hasher.Sum(nil)) + packKey = fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, packID[:2], packID) + for chunkID, location := range chunks { + location.Pack = packKey + chunks[chunkID] = location + } + segment := repositoryIndexSegment{Version: repositoryFormatVersion, CreatedAt: time.Now().UTC(), Pack: packKey, Chunks: chunks} + indexBytes, err := json.Marshal(segment) + if err != nil { + return "", 0, 0, fmt.Errorf("encode repository index: %w", err) + } + if _, err := pack.Seek(0, io.SeekStart); err != nil { + return "", 0, 0, fmt.Errorf("rewind repository pack: %w", err) + } + if err := provider.Upload(ctx, packKey, pack, packBytes, map[string]string{"format": "backupx-pack-v1"}); err != nil { + return "", 0, 0, fmt.Errorf("upload repository pack %s: %w", packKey, err) + } + closeErr := pack.Close() + packClosed = true + if closeErr != nil { + return "", 0, 0, fmt.Errorf("close repository pack: %w", closeErr) + } + indexKey := fmt.Sprintf("%s/%s.json", repositoryIndexPrefix, packID) + if err := provider.Upload(ctx, indexKey, bytes.NewReader(indexBytes), int64(len(indexBytes)), map[string]string{"format": "backupx-index-v1"}); err != nil { + deleteErr := provider.Delete(ctx, packKey) + return "", 0, 0, errors.Join(fmt.Errorf("upload repository index %s: %w", indexKey, err), deleteErr) + } + return packKey, int64(len(indexBytes)), packBytes, nil +} + +func (s *RepositoryStore) loadIndex(ctx context.Context, provider storage.StorageProvider) (map[string]repositoryChunkLocation, error) { + objects, err := provider.List(ctx, repositoryIndexPrefix) + if err != nil { + return nil, fmt.Errorf("list repository indexes: %w", err) + } + sort.Slice(objects, func(i, j int) bool { return objects[i].Key < objects[j].Key }) + locations := make(map[string]repositoryChunkLocation) + for _, object := range objects { + segment, readErr := s.readIndexSegment(ctx, provider, object.Key) + if readErr != nil { + return nil, readErr + } + for chunkID, location := range segment.Chunks { + if _, exists := locations[chunkID]; !exists { + locations[chunkID] = location + } + } + } + return locations, nil +} + +func (s *RepositoryStore) readIndexSegment(ctx context.Context, provider storage.StorageProvider, key string) (*repositoryIndexSegment, error) { + indexPrefix := repositoryIndexPrefix + "/" + indexName := strings.TrimPrefix(key, indexPrefix) + indexID := strings.TrimSuffix(indexName, ".json") + decodedIndexID, decodeIndexErr := hex.DecodeString(indexID) + if indexName == key || indexID == indexName || indexID != strings.ToLower(indexID) || strings.Contains(indexName, "/") || strings.Contains(indexName, "\\") || decodeIndexErr != nil || len(decodedIndexID) != sha256.Size { + return nil, fmt.Errorf("invalid repository index key %q", key) + } + expectedPack := fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, indexID[:2], indexID) + reader, err := provider.Download(ctx, key) + if err != nil { + return nil, fmt.Errorf("download repository index %s: %w", key, err) + } + data, readErr := io.ReadAll(io.LimitReader(reader, repositoryMaxIndexSize+1)) + closeErr := reader.Close() + if readErr != nil { + return nil, fmt.Errorf("read repository index %s: %w", key, readErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close repository index %s: %w", key, closeErr) + } + if int64(len(data)) > repositoryMaxIndexSize { + return nil, fmt.Errorf("repository index %s exceeds %d bytes", key, repositoryMaxIndexSize) + } + var segment repositoryIndexSegment + if err := json.Unmarshal(data, &segment); err != nil { + return nil, fmt.Errorf("decode repository index %s: %w", key, err) + } + if segment.Version != repositoryFormatVersion || segment.Pack != expectedPack || len(segment.Chunks) == 0 { + return nil, fmt.Errorf("unsupported repository index %s", key) + } + packLimit := s.packSize + if packLimit <= 0 { + packLimit = repositoryDefaultPack + } + if packLimit < repositoryMaxEncoded { + packLimit = repositoryMaxEncoded + } + for chunkID, location := range segment.Chunks { + expectedPrefix := "p-" + if location.Encrypted { + expectedPrefix = "e-" + } + encodedID := strings.TrimPrefix(chunkID, expectedPrefix) + decodedID, decodeErr := hex.DecodeString(encodedID) + validPrefix := encodedID != chunkID + validCompression := location.Compression == "none" || location.Compression == "gzip" || location.Compression == "zstd" + if len(decodedID) != sha256.Size || decodeErr != nil || encodedID != strings.ToLower(encodedID) || !validPrefix || !validCompression || location.Pack != expectedPack || location.Offset < 0 || location.Length <= 0 || location.Length > repositoryMaxEncoded || location.Length > packLimit || location.PlainSize <= 0 || location.PlainSize > repositoryChunkMax || location.Offset > packLimit-location.Length { + return nil, fmt.Errorf("invalid chunk location in repository index %s", key) + } + } + return &segment, nil +} + +func (s *RepositoryStore) loadSnapshot(ctx context.Context, provider storage.StorageProvider, key, expectedChecksum string) (*repositorySnapshot, []byte, error) { + reader, err := provider.Download(ctx, key) + if err != nil { + return nil, nil, fmt.Errorf("download repository snapshot %s: %w", key, err) + } + data, readErr := io.ReadAll(io.LimitReader(reader, repositoryMaxSnapshot+1)) + closeErr := reader.Close() + if readErr != nil { + return nil, nil, fmt.Errorf("read repository snapshot %s: %w", key, readErr) + } + if closeErr != nil { + return nil, nil, fmt.Errorf("close repository snapshot %s: %w", key, closeErr) + } + if int64(len(data)) > repositoryMaxSnapshot { + return nil, nil, fmt.Errorf("repository snapshot %s exceeds %d bytes", key, repositoryMaxSnapshot) + } + if expected := strings.TrimSpace(expectedChecksum); expected != "" { + expectedBytes, decodeErr := hex.DecodeString(expected) + if decodeErr != nil || len(expectedBytes) != sha256.Size { + return nil, nil, fmt.Errorf("repository snapshot checksum is invalid") + } + digest := sha256.Sum256(data) + if !strings.EqualFold(hex.EncodeToString(digest[:]), expected) { + return nil, nil, fmt.Errorf("repository snapshot checksum mismatch") + } + } + var envelope repositorySnapshotEnvelope + if err := json.Unmarshal(data, &envelope); err != nil { + return nil, nil, fmt.Errorf("decode repository snapshot envelope %s: %w", key, err) + } + if envelope.Version != repositoryFormatVersion { + return nil, nil, fmt.Errorf("unsupported repository snapshot version %d", envelope.Version) + } + payload := []byte(envelope.Data) + if envelope.Encrypted { + ciphertext, decodeErr := base64.RawURLEncoding.DecodeString(envelope.Ciphertext) + if decodeErr != nil { + return nil, nil, fmt.Errorf("decode repository snapshot ciphertext: %w", decodeErr) + } + payload, err = s.decrypt(ciphertext, []byte("backupx-repository-snapshot-v1")) + if err != nil { + return nil, nil, fmt.Errorf("decrypt repository snapshot: %w", err) + } + } + var snapshot repositorySnapshot + if err := json.Unmarshal(payload, &snapshot); err != nil { + return nil, nil, fmt.Errorf("decode repository snapshot: %w", err) + } + if snapshot.Version != repositoryFormatVersion { + return nil, nil, fmt.Errorf("unsupported repository snapshot payload version %d", snapshot.Version) + } + if snapshot.Encrypted != envelope.Encrypted { + return nil, nil, fmt.Errorf("repository snapshot encryption metadata mismatch") + } + if err := s.validateSnapshot(&snapshot); err != nil { + return nil, nil, err + } + return &snapshot, data, nil +} + +func (*RepositoryStore) validateSnapshot(snapshot *repositorySnapshot) error { + if snapshot == nil { + return fmt.Errorf("repository snapshot is required") + } + if snapshot.Version != repositoryFormatVersion { + return fmt.Errorf("unsupported repository snapshot payload version %d", snapshot.Version) + } + if snapshot.Compression != "none" && snapshot.Compression != "gzip" && snapshot.Compression != "zstd" { + return fmt.Errorf("unsupported repository snapshot compression %q", snapshot.Compression) + } + seenPaths := make(map[string]struct{}, len(snapshot.Entries)) + symlinkPaths := make(map[string]struct{}) + for _, entry := range snapshot.Entries { + if entry.Path == "." || !fs.ValidPath(entry.Path) || path.Clean(entry.Path) != entry.Path || strings.Contains(entry.Path, "\\") || entry.Mode & ^uint32(0o777) != 0 || entry.Size < 0 { + return fmt.Errorf("invalid repository snapshot entry %q", entry.Path) + } + if _, exists := seenPaths[entry.Path]; exists { + return fmt.Errorf("duplicate repository snapshot entry %q", entry.Path) + } + seenPaths[entry.Path] = struct{}{} + switch entry.Kind { + case "directory": + if len(entry.Chunks) != 0 || entry.LinkTarget != "" { + return fmt.Errorf("invalid repository directory entry %q", entry.Path) + } + case "symlink": + if entry.LinkTarget == "" || len(entry.Chunks) != 0 { + return fmt.Errorf("invalid repository symlink entry %q", entry.Path) + } + normalizedTarget := strings.ReplaceAll(entry.LinkTarget, "\\", "/") + resolvedTarget := path.Clean(path.Join(path.Dir(entry.Path), normalizedTarget)) + if strings.ContainsRune(entry.LinkTarget, 0) || path.IsAbs(normalizedTarget) || filepath.IsAbs(entry.LinkTarget) || filepath.VolumeName(entry.LinkTarget) != "" || resolvedTarget == ".." || strings.HasPrefix(resolvedTarget, "../") { + return fmt.Errorf("repository symlink %q escapes the restore root", entry.Path) + } + symlinkPaths[entry.Path] = struct{}{} + case "file": + if entry.LinkTarget != "" || (entry.Size == 0 && len(entry.Chunks) != 0) || (entry.Size > 0 && len(entry.Chunks) == 0) { + return fmt.Errorf("invalid repository file entry %q", entry.Path) + } + expectedPrefix := "p-" + if snapshot.Encrypted { + expectedPrefix = "e-" + } + for _, chunkID := range entry.Chunks { + encodedID := strings.TrimPrefix(chunkID, expectedPrefix) + decodedID, decodeErr := hex.DecodeString(encodedID) + if decodeErr != nil || len(decodedID) != sha256.Size || encodedID == chunkID || encodedID != strings.ToLower(encodedID) { + return fmt.Errorf("invalid chunk id in repository entry %q", entry.Path) + } + } + default: + return fmt.Errorf("unsupported repository entry kind %q", entry.Kind) + } + } + for entryPath := range seenPaths { + for parent := path.Dir(entryPath); parent != "."; parent = path.Dir(parent) { + if _, crossesSymlink := symlinkPaths[parent]; crossesSymlink { + return fmt.Errorf("repository entry %q crosses symlink %q", entryPath, parent) + } + } + } + return nil +} + +func (s *RepositoryStore) encodeSnapshot(snapshot repositorySnapshot) ([]byte, error) { + payload, err := json.Marshal(snapshot) + if err != nil { + return nil, fmt.Errorf("encode repository snapshot: %w", err) + } + envelope := repositorySnapshotEnvelope{Version: repositoryFormatVersion, Encrypted: snapshot.Encrypted} + if snapshot.Encrypted { + ciphertext, encryptErr := s.encrypt(payload, []byte("backupx-repository-snapshot-v1")) + if encryptErr != nil { + return nil, fmt.Errorf("encrypt repository snapshot: %w", encryptErr) + } + envelope.Ciphertext = base64.RawURLEncoding.EncodeToString(ciphertext) + } else { + envelope.Data = payload + } + data, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("encode repository snapshot envelope: %w", err) + } + return data, nil +} + +func (s *RepositoryStore) readChunk(ctx context.Context, provider storage.StorageProvider, chunkID string, locations map[string]repositoryChunkLocation) ([]byte, error) { + location, exists := locations[chunkID] + if !exists { + return nil, fmt.Errorf("repository chunk %s is missing from the index", chunkID) + } + var reader io.ReadCloser + var err error + if ranged, ok := provider.(storage.StorageRangeDownloader); ok { + reader, err = ranged.DownloadRange(ctx, location.Pack, location.Offset, location.Length) + } else { + reader, err = provider.Download(ctx, location.Pack) + if err == nil && location.Offset > 0 { + if _, copyErr := io.CopyN(io.Discard, reader, location.Offset); copyErr != nil { + return nil, errors.Join(fmt.Errorf("seek repository pack %s: %w", location.Pack, copyErr), reader.Close()) + } + } + } + if err != nil { + return nil, fmt.Errorf("read repository pack %s: %w", location.Pack, err) + } + encoded := make([]byte, int(location.Length)) + _, readErr := io.ReadFull(reader, encoded) + closeErr := reader.Close() + if readErr != nil { + return nil, fmt.Errorf("read repository chunk %s: %w", chunkID, readErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close repository pack %s: %w", location.Pack, closeErr) + } + raw, err := s.decodeChunk(encoded, location, chunkID) + if err != nil { + return nil, err + } + if int64(len(raw)) != location.PlainSize { + return nil, fmt.Errorf("repository chunk %s size mismatch", chunkID) + } + if actual := s.chunkID(raw, location.Compression, location.Encrypted); actual != chunkID { + return nil, fmt.Errorf("repository chunk %s failed content verification", chunkID) + } + return raw, nil +} + +func (s *RepositoryStore) encodeChunk(raw []byte, compression string, encrypted bool, chunkID string) ([]byte, error) { + var encoded []byte + switch compression { + case "none": + encoded = append([]byte(nil), raw...) + case "gzip": + var buffer bytes.Buffer + writer := gzip.NewWriter(&buffer) + if _, err := writer.Write(raw); err != nil { + return nil, errors.Join(fmt.Errorf("gzip repository chunk: %w", err), writer.Close()) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("close repository gzip chunk: %w", err) + } + encoded = buffer.Bytes() + case "zstd": + writer, err := zstd.NewWriter(nil) + if err != nil { + return nil, fmt.Errorf("create repository zstd encoder: %w", err) + } + encoded = writer.EncodeAll(raw, nil) + writer.Close() + default: + return nil, fmt.Errorf("unsupported repository compression %q", compression) + } + if !encrypted { + return encoded, nil + } + ciphertext, err := s.encrypt(encoded, []byte(chunkID)) + if err != nil { + return nil, fmt.Errorf("encrypt repository chunk: %w", err) + } + return ciphertext, nil +} + +func (s *RepositoryStore) decodeChunk(encoded []byte, location repositoryChunkLocation, chunkID string) ([]byte, error) { + payload := encoded + var err error + if location.Encrypted { + payload, err = s.decrypt(encoded, []byte(chunkID)) + if err != nil { + return nil, fmt.Errorf("decrypt repository chunk %s: %w", chunkID, err) + } + } + switch location.Compression { + case "none": + return payload, nil + case "gzip": + reader, err := gzip.NewReader(bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("open repository gzip chunk: %w", err) + } + raw, readErr := io.ReadAll(io.LimitReader(reader, location.PlainSize+1)) + closeErr := reader.Close() + if readErr != nil { + return nil, fmt.Errorf("decompress repository gzip chunk: %w", readErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close repository gzip chunk: %w", closeErr) + } + return raw, nil + case "zstd": + reader, err := zstd.NewReader(bytes.NewReader(payload), zstd.WithDecoderMaxMemory(uint64(repositoryChunkMax+(1<<20)))) + if err != nil { + return nil, fmt.Errorf("create repository zstd decoder: %w", err) + } + raw, readErr := io.ReadAll(io.LimitReader(reader, location.PlainSize+1)) + reader.Close() + if readErr != nil { + return nil, fmt.Errorf("decompress repository zstd chunk: %w", readErr) + } + return raw, nil + default: + return nil, fmt.Errorf("unsupported repository compression %q", location.Compression) + } +} + +func (s *RepositoryStore) chunkID(raw []byte, compression string, encrypted bool) string { + domain := fmt.Sprintf("backupx-repository-v1/%s/plain\x00", compression) + prefix := "p-" + if encrypted { + domain = fmt.Sprintf("backupx-repository-v1/%s/encrypted\x00", compression) + mac := hmac.New(sha256.New, s.key) + mac.Write([]byte(domain)) + mac.Write(raw) + return "e-" + hex.EncodeToString(mac.Sum(nil)) + } + digest := sha256.New() + digest.Write([]byte(domain)) + digest.Write(raw) + return prefix + hex.EncodeToString(digest.Sum(nil)) +} + +func (s *RepositoryStore) encrypt(plain, additionalData []byte) ([]byte, error) { + if len(s.key) != 32 { + return nil, fmt.Errorf("repository encryption key is unavailable") + } + block, err := aes.NewCipher(s.key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, plain, additionalData), nil +} + +func (s *RepositoryStore) decrypt(ciphertext, additionalData []byte) ([]byte, error) { + if len(s.key) != 32 { + return nil, fmt.Errorf("repository encryption key is unavailable") + } + block, err := aes.NewCipher(s.key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + if len(ciphertext) < gcm.NonceSize() { + return nil, fmt.Errorf("repository ciphertext is too short") + } + nonce := ciphertext[:gcm.NonceSize()] + return gcm.Open(nil, nonce, ciphertext[gcm.NonceSize():], additionalData) +} + +func (s *RepositoryStore) normalizeCompression(value string) (string, error) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "zstd": + return "zstd", nil + case "gzip": + return "gzip", nil + case "none": + return "none", nil + default: + return "", fmt.Errorf("unsupported repository compression %q", value) + } +} + +func (s *RepositoryStore) openRestoreRoot(target string) (*os.Root, error) { + cleanTarget := filepath.Clean(target) + if !filepath.IsAbs(cleanTarget) { + return nil, fmt.Errorf("restore target must be absolute: %s", target) + } + volume := filepath.VolumeName(cleanTarget) + if strings.Contains(volume, "..") || strings.ContainsRune(volume, 0) { + return nil, fmt.Errorf("invalid restore target volume: %s", target) + } + volumeRootPath := string(filepath.Separator) + relativeTarget := strings.TrimLeft(cleanTarget, string(filepath.Separator)) + if volume != "" { + volumeRootPath = volume + string(filepath.Separator) + relativeTarget = strings.TrimLeft(strings.TrimPrefix(cleanTarget, volume), string(filepath.Separator)) + } + if relativeTarget != "" && !filepath.IsLocal(relativeTarget) { + return nil, fmt.Errorf("restore target is not local to its volume: %s", target) + } + volumeRoot, err := os.OpenRoot(volumeRootPath) + if err != nil { + return nil, fmt.Errorf("open restore volume: %w", err) + } + if relativeTarget == "" { + return volumeRoot, nil + } + if err := volumeRoot.MkdirAll(relativeTarget, 0o755); err != nil { + return nil, errors.Join(fmt.Errorf("create repository restore root: %w", err), volumeRoot.Close()) + } + restoreRoot, openErr := volumeRoot.OpenRoot(relativeTarget) + closeErr := volumeRoot.Close() + if openErr != nil || closeErr != nil { + return nil, errors.Join(openErr, closeErr) + } + return restoreRoot, nil +} + +func compactPaths(items []string) []string { + result := make([]string, 0, len(items)) + for _, item := range items { + if value := strings.TrimSpace(item); value != "" { + result = append(result, filepath.Clean(value)) + } + } + return result +} diff --git a/server/internal/backup/repository_chunker.go b/server/internal/backup/repository_chunker.go new file mode 100644 index 0000000..1dabf21 --- /dev/null +++ b/server/internal/backup/repository_chunker.go @@ -0,0 +1,131 @@ +package backup + +import ( + "context" + "fmt" + "io" +) + +const ( + repositoryChunkMin = 512 << 10 + repositoryChunkAvg = 1 << 20 + repositoryChunkMax = 4 << 20 +) + +// contentDefinedChunker implements the normalized FastCDC cut-point strategy. +// The rolling Gear hash only retains the latest 64 bytes through uint64 +// overflow, so chunk boundaries re-synchronize after insertions or deletions. +type contentDefinedChunker struct { + minSize int + avgSize int + maxSize int + smallMask uint64 + largeMask uint64 + gear [256]uint64 +} + +func newContentDefinedChunker() *contentDefinedChunker { + chunker := &contentDefinedChunker{ + minSize: repositoryChunkMin, + avgSize: repositoryChunkAvg, + maxSize: repositoryChunkMax, + smallMask: (1 << 21) - 1, + largeMask: (1 << 19) - 1, + } + + // SplitMix64 produces a stable, well-distributed Gear table. The seed and + // generation algorithm are part of repository format v1 and must not change. + seed := uint64(0x6a09e667f3bcc909) + for i := range chunker.gear { + seed += 0x9e3779b97f4a7c15 + value := seed + value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9 + value = (value ^ (value >> 27)) * 0x94d049bb133111eb + chunker.gear[i] = value ^ (value >> 31) + } + return chunker +} + +func (c *contentDefinedChunker) Split(ctx context.Context, reader io.Reader, emit func([]byte) error) error { + if reader == nil || emit == nil { + return fmt.Errorf("chunk reader and emitter are required") + } + + pending := make([]byte, 0, c.maxSize+(256<<10)) + readBuffer := make([]byte, 256<<10) + eof := false + for { + if !eof { + readCount, readErr := reader.Read(readBuffer) + if readCount > 0 { + pending = append(pending, readBuffer[:readCount]...) + } + switch readErr { + case nil: + case io.EOF: + eof = true + default: + return fmt.Errorf("read source for chunking: %w", readErr) + } + if readCount == 0 && readErr == nil { + continue + } + } + + for len(pending) > 0 { + if err := ctx.Err(); err != nil { + return err + } + cut := c.findCutPoint(pending, eof) + if cut == 0 { + break + } + chunk := make([]byte, cut) + copy(chunk, pending[:cut]) + if err := emit(chunk); err != nil { + return err + } + copy(pending, pending[cut:]) + pending = pending[:len(pending)-cut] + } + + if eof { + if len(pending) != 0 { + return fmt.Errorf("chunker stopped with %d buffered bytes", len(pending)) + } + return nil + } + } +} + +func (c *contentDefinedChunker) findCutPoint(data []byte, eof bool) int { + if len(data) < c.minSize { + if eof { + return len(data) + } + return 0 + } + + limit := len(data) + if limit > c.maxSize { + limit = c.maxSize + } + var hash uint64 + for index := c.minSize; index < limit; index++ { + hash = (hash << 1) + c.gear[data[index]] + mask := c.largeMask + if index < c.avgSize { + mask = c.smallMask + } + if hash&mask == 0 { + return index + 1 + } + } + if len(data) >= c.maxSize { + return c.maxSize + } + if eof { + return len(data) + } + return 0 +} diff --git a/server/internal/backup/repository_test.go b/server/internal/backup/repository_test.go new file mode 100644 index 0000000..282e39d --- /dev/null +++ b/server/internal/backup/repository_test.go @@ -0,0 +1,414 @@ +package backup + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "backupx/server/internal/storage" +) + +func TestContentDefinedChunkerResynchronizesAfterInsertion(t *testing.T) { + source := make([]byte, 8<<20) + if _, err := rand.New(rand.NewSource(42)).Read(source); err != nil { + t.Fatalf("generate source: %v", err) + } + modified := make([]byte, 0, len(source)+4096) + modified = append(modified, source[:2<<20]...) + modified = append(modified, bytes.Repeat([]byte("inserted"), 512)...) + modified = append(modified, source[2<<20:]...) + + chunker := newContentDefinedChunker() + collect := func(data []byte) map[string]struct{} { + t.Helper() + ids := make(map[string]struct{}) + err := chunker.Split(context.Background(), bytes.NewReader(data), func(chunk []byte) error { + digest := sha256.Sum256(chunk) + ids[fmt.Sprintf("%x", digest[:])] = struct{}{} + if len(chunk) > repositoryChunkMax { + return fmt.Errorf("chunk exceeds maximum: %d", len(chunk)) + } + return nil + }) + if err != nil { + t.Fatalf("split chunks: %v", err) + } + return ids + } + originalChunks := collect(source) + modifiedChunks := collect(modified) + shared := 0 + for chunkID := range originalChunks { + if _, ok := modifiedChunks[chunkID]; ok { + shared++ + } + } + if shared < len(originalChunks)/2 { + t.Fatalf("content-defined boundaries did not resynchronize: shared=%d original=%d", shared, len(originalChunks)) + } +} + +func TestRepositoryRoundTripDedupAndPrune(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + sourceDir := filepath.Join(tempDir, "dataset") + if err := os.MkdirAll(filepath.Join(sourceDir, "empty"), 0o755); err != nil { + t.Fatalf("create source: %v", err) + } + original := make([]byte, 6<<20) + if _, err := rand.New(rand.NewSource(7)).Read(original); err != nil { + t.Fatalf("generate fixture: %v", err) + } + primaryPath := filepath.Join(sourceDir, "primary.bin") + duplicatePath := filepath.Join(sourceDir, "duplicate.bin") + if err := os.WriteFile(primaryPath, original, 0o640); err != nil { + t.Fatalf("write primary: %v", err) + } + if err := os.WriteFile(duplicatePath, original, 0o640); err != nil { + t.Fatalf("write duplicate: %v", err) + } + + key := sha256.Sum256([]byte("repository-test-key")) + store := NewRepositoryStore(key[:]) + provider := newMemoryRepositoryProvider() + task := TaskSpec{ + ID: 12, + Name: "repository-test", + Type: "file", + SourcePaths: []string{sourceDir}, + Compression: "zstd", + Encrypt: true, + StartedAt: time.Date(2026, 8, 6, 1, 2, 3, 0, time.UTC), + TempDir: tempDir, + } + + firstPlan, err := store.BuildPlan(ctx, task, NopLogWriter{}) + if err != nil { + t.Fatalf("build first plan: %v", err) + } + firstKey := store.SnapshotKey(task.ID, 1, task.StartedAt) + firstResult, err := store.Upload(ctx, provider, firstPlan, firstKey) + if closeErr := firstPlan.Close(); closeErr != nil { + t.Fatalf("close first plan: %v", closeErr) + } + if err != nil { + t.Fatalf("upload first snapshot: %v", err) + } + if firstResult.NewChunks == 0 || firstResult.UniqueChunks == 0 { + t.Fatalf("first upload did not create chunks: %+v", firstResult) + } + if firstPlan.UniqueSize >= firstPlan.LogicalSize { + t.Fatalf("duplicate file was not deduplicated within plan: unique=%d logical=%d", firstPlan.UniqueSize, firstPlan.LogicalSize) + } + + modified := make([]byte, 0, len(original)+4096) + modified = append(modified, original[:2<<20]...) + modified = append(modified, bytes.Repeat([]byte("changed!"), 512)...) + modified = append(modified, original[2<<20:]...) + if err := os.WriteFile(primaryPath, modified, 0o640); err != nil { + t.Fatalf("modify primary: %v", err) + } + task.StartedAt = task.StartedAt.Add(time.Hour) + secondPlan, err := store.BuildPlan(ctx, task, NopLogWriter{}) + if err != nil { + t.Fatalf("build second plan: %v", err) + } + secondKey := store.SnapshotKey(task.ID, 2, task.StartedAt) + secondResult, err := store.Upload(ctx, provider, secondPlan, secondKey) + if closeErr := secondPlan.Close(); closeErr != nil { + t.Fatalf("close second plan: %v", closeErr) + } + if err != nil { + t.Fatalf("upload second snapshot: %v", err) + } + if secondResult.ReusedBytes <= secondResult.LogicalSize/3 { + t.Fatalf("second snapshot reused too little data: %+v", secondResult) + } + if secondResult.UploadedBytes >= secondResult.LogicalSize { + t.Fatalf("incremental upload was not smaller than logical data: %+v", secondResult) + } + + verify, err := store.Verify(ctx, provider, secondKey, secondResult.Checksum) + if err != nil { + t.Fatalf("verify repository: %v", err) + } + if verify.Chunks == 0 || verify.Bytes == 0 { + t.Fatalf("empty verification result: %+v", verify) + } + + restoreRoot := filepath.Join(tempDir, "restore") + restoreTask := task + restoreTask.RestoreTargetPath = restoreRoot + if err := store.Restore(ctx, provider, secondKey, strings.Repeat("0", sha256.Size*2), restoreTask, NopLogWriter{}); err == nil { + t.Fatal("restore accepted a mismatched snapshot checksum") + } + if err := store.Restore(ctx, provider, secondKey, "", restoreTask, NopLogWriter{}); err == nil { + t.Fatal("restore accepted a missing snapshot checksum") + } + if err := store.Restore(ctx, provider, secondKey, secondResult.Checksum, restoreTask, NopLogWriter{}); err != nil { + t.Fatalf("restore snapshot: %v", err) + } + restored, err := os.ReadFile(filepath.Join(restoreRoot, filepath.Base(sourceDir), "primary.bin")) + if err != nil { + t.Fatalf("read restored primary: %v", err) + } + if !bytes.Equal(restored, modified) { + t.Fatalf("restored primary differs from source") + } + if info, err := os.Stat(filepath.Join(restoreRoot, filepath.Base(sourceDir), "empty")); err != nil || !info.IsDir() { + t.Fatalf("empty directory was not restored: info=%v err=%v", info, err) + } + + if err := provider.Delete(ctx, firstKey); err != nil { + t.Fatalf("delete first snapshot: %v", err) + } + if _, err := store.Prune(ctx, provider); err != nil { + t.Fatalf("prune with live snapshot: %v", err) + } + if err := provider.Delete(ctx, secondKey); err != nil { + t.Fatalf("delete second snapshot: %v", err) + } + pruned, err := store.Prune(ctx, provider) + if err != nil { + t.Fatalf("prune empty repository: %v", err) + } + if pruned.DeletedPacks == 0 || pruned.DeletedIndexes == 0 { + t.Fatalf("prune did not reclaim repository data: %+v", pruned) + } + if objects, err := provider.List(ctx, repositoryPackPrefix); err != nil || len(objects) != 0 { + t.Fatalf("packs remain after prune: objects=%v err=%v", objects, err) + } +} + +func TestRepositoryRestoreRejectsUnsafeSnapshotMetadata(t *testing.T) { + cases := []struct { + name string + entries []repositoryEntry + }{ + { + name: "path traversal", + entries: []repositoryEntry{{Path: "../escape", Kind: "directory", Mode: 0o755}}, + }, + { + name: "entry below symlink", + entries: []repositoryEntry{ + {Path: "link", Kind: "symlink", Mode: 0o777, LinkTarget: "inside"}, + {Path: "link/payload", Kind: "file", Mode: 0o600, Size: 1, Chunks: []string{"p-" + strings.Repeat("0", sha256.Size*2)}}, + }, + }, + { + name: "escaping symlink target", + entries: []repositoryEntry{{Path: "escape", Kind: "symlink", Mode: 0o777, LinkTarget: "../outside"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + store := NewRepositoryStore(nil) + provider := newMemoryRepositoryProvider() + snapshot := repositorySnapshot{ + Version: repositoryFormatVersion, + TaskID: 1, + CreatedAt: time.Now().UTC(), + Compression: "none", + Entries: tc.entries, + } + data, err := store.encodeSnapshot(snapshot) + if err != nil { + t.Fatalf("encodeSnapshot returned error: %v", err) + } + key := store.SnapshotKey(1, 1, snapshot.CreatedAt) + if err := provider.Upload(ctx, key, bytes.NewReader(data), int64(len(data)), nil); err != nil { + t.Fatalf("Upload snapshot returned error: %v", err) + } + digest := sha256.Sum256(data) + task := TaskSpec{SourcePath: filepath.Join(t.TempDir(), "source"), RestoreTargetPath: filepath.Join(t.TempDir(), "restore")} + if err := store.Restore(ctx, provider, key, fmt.Sprintf("%x", digest[:]), task, NopLogWriter{}); err == nil { + t.Fatal("restore accepted unsafe snapshot metadata") + } + }) + } +} + +func TestRepositoryRestorePreservesDirectoryWhenSnapshotContainsSymlink(t *testing.T) { + ctx := context.Background() + store := NewRepositoryStore(nil) + provider := newMemoryRepositoryProvider() + snapshot := repositorySnapshot{ + Version: repositoryFormatVersion, + TaskID: 1, + CreatedAt: time.Now().UTC(), + Compression: "none", + Entries: []repositoryEntry{{Path: "link", Kind: "symlink", Mode: 0o777, LinkTarget: "inside"}}, + } + data, err := store.encodeSnapshot(snapshot) + if err != nil { + t.Fatalf("encodeSnapshot returned error: %v", err) + } + key := store.SnapshotKey(1, 1, snapshot.CreatedAt) + if err := provider.Upload(ctx, key, bytes.NewReader(data), int64(len(data)), nil); err != nil { + t.Fatalf("Upload snapshot returned error: %v", err) + } + digest := sha256.Sum256(data) + restoreRoot := filepath.Join(t.TempDir(), "restore") + markerPath := filepath.Join(restoreRoot, "link", "keep.txt") + if err := os.MkdirAll(filepath.Dir(markerPath), 0o755); err != nil { + t.Fatalf("MkdirAll marker parent: %v", err) + } + if err := os.WriteFile(markerPath, []byte("keep"), 0o600); err != nil { + t.Fatalf("WriteFile marker: %v", err) + } + task := TaskSpec{SourcePath: filepath.Join(t.TempDir(), "source"), RestoreTargetPath: restoreRoot} + if err := store.Restore(ctx, provider, key, fmt.Sprintf("%x", digest[:]), task, NopLogWriter{}); err == nil { + t.Fatal("restore replaced an existing directory with a symlink") + } + if data, err := os.ReadFile(markerPath); err != nil || string(data) != "keep" { + t.Fatalf("existing directory content changed: data=%q err=%v", data, err) + } +} + +func TestRepositoryRejectsOversizedChunkLocation(t *testing.T) { + ctx := context.Background() + store := NewRepositoryStore(nil) + provider := newMemoryRepositoryProvider() + chunkID := "p-" + strings.Repeat("0", sha256.Size*2) + packID := strings.Repeat("a", sha256.Size*2) + segment := repositoryIndexSegment{ + Version: repositoryFormatVersion, + Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, packID[:2], packID), + Chunks: map[string]repositoryChunkLocation{ + chunkID: {Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, packID[:2], packID), Offset: 0, Length: repositoryMaxEncoded + 1, PlainSize: 1, Compression: "none"}, + }, + } + data, err := json.Marshal(segment) + if err != nil { + t.Fatalf("Marshal index returned error: %v", err) + } + indexKey := fmt.Sprintf("%s/%s.json", repositoryIndexPrefix, packID) + if err := provider.Upload(ctx, indexKey, bytes.NewReader(data), int64(len(data)), nil); err != nil { + t.Fatalf("Upload index returned error: %v", err) + } + if _, err := store.loadIndex(ctx, provider); err == nil { + t.Fatal("loadIndex accepted an oversized encoded chunk") + } +} + +func TestRepositoryRejectsIndexPackMismatch(t *testing.T) { + ctx := context.Background() + store := NewRepositoryStore(nil) + provider := newMemoryRepositoryProvider() + chunkID := "p-" + strings.Repeat("0", sha256.Size*2) + indexID := strings.Repeat("a", sha256.Size*2) + otherPackID := strings.Repeat("b", sha256.Size*2) + expectedPack := fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, indexID[:2], indexID) + segment := repositoryIndexSegment{ + Version: repositoryFormatVersion, + Pack: expectedPack, + Chunks: map[string]repositoryChunkLocation{ + chunkID: { + Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, otherPackID[:2], otherPackID), + Offset: 0, + Length: 1, + PlainSize: 1, + Compression: "none", + }, + }, + } + data, err := json.Marshal(segment) + if err != nil { + t.Fatalf("Marshal index returned error: %v", err) + } + indexKey := fmt.Sprintf("%s/%s.json", repositoryIndexPrefix, indexID) + if err := provider.Upload(ctx, indexKey, bytes.NewReader(data), int64(len(data)), nil); err != nil { + t.Fatalf("Upload index returned error: %v", err) + } + if _, err := store.loadIndex(ctx, provider); err == nil { + t.Fatal("loadIndex accepted a chunk location pointing to a different pack") + } +} + +type memoryRepositoryProvider struct { + mu sync.RWMutex + objects map[string][]byte + times map[string]time.Time +} + +func newMemoryRepositoryProvider() *memoryRepositoryProvider { + return &memoryRepositoryProvider{objects: make(map[string][]byte), times: make(map[string]time.Time)} +} + +func (p *memoryRepositoryProvider) Type() storage.ProviderType { return "memory" } +func (p *memoryRepositoryProvider) TestConnection(context.Context) error { return nil } + +func (p *memoryRepositoryProvider) Upload(_ context.Context, key string, reader io.Reader, size int64, _ map[string]string) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + if int64(len(data)) != size { + return fmt.Errorf("size mismatch for %s: %d != %d", key, len(data), size) + } + p.mu.Lock() + defer p.mu.Unlock() + p.objects[key] = append([]byte(nil), data...) + p.times[key] = time.Now().UTC() + return nil +} + +func (p *memoryRepositoryProvider) Download(_ context.Context, key string) (io.ReadCloser, error) { + p.mu.RLock() + defer p.mu.RUnlock() + data, ok := p.objects[key] + if !ok { + return nil, fmt.Errorf("object %s not found", key) + } + return io.NopCloser(bytes.NewReader(append([]byte(nil), data...))), nil +} + +func (p *memoryRepositoryProvider) DownloadRange(_ context.Context, key string, offset, length int64) (io.ReadCloser, error) { + p.mu.RLock() + defer p.mu.RUnlock() + data, ok := p.objects[key] + if !ok { + return nil, fmt.Errorf("object %s not found", key) + } + if offset < 0 || length <= 0 || offset+length > int64(len(data)) { + return nil, fmt.Errorf("invalid range %d:%d for %s", offset, length, key) + } + return io.NopCloser(bytes.NewReader(append([]byte(nil), data[offset:offset+length]...))), nil +} + +func (p *memoryRepositoryProvider) Delete(_ context.Context, key string) error { + p.mu.Lock() + defer p.mu.Unlock() + if _, ok := p.objects[key]; !ok { + return fmt.Errorf("object %s not found", key) + } + delete(p.objects, key) + delete(p.times, key) + return nil +} + +func (p *memoryRepositoryProvider) List(_ context.Context, prefix string) ([]storage.ObjectInfo, error) { + p.mu.RLock() + defer p.mu.RUnlock() + result := make([]storage.ObjectInfo, 0) + for key, data := range p.objects { + if strings.HasPrefix(key, prefix) { + result = append(result, storage.ObjectInfo{Key: key, Size: int64(len(data)), UpdatedAt: p.times[key]}) + } + } + sort.Slice(result, func(i, j int) bool { return result[i].Key < result[j].Key }) + return result, nil +} diff --git a/server/internal/backup/retention/service.go b/server/internal/backup/retention/service.go index 70dda96..6e632c9 100644 --- a/server/internal/backup/retention/service.go +++ b/server/internal/backup/retention/service.go @@ -2,11 +2,13 @@ package retention import ( "context" + "encoding/json" "fmt" "sort" "strings" "time" + "backupx/server/internal/backup" "backupx/server/internal/model" "backupx/server/internal/repository" "backupx/server/internal/storage" @@ -40,16 +42,48 @@ type CleanupResult struct { Warnings []string } +type cleanupObject struct { + targetID uint + path string +} + +type storedUploadResult struct { + StorageTargetID uint `json:"storageTargetId"` + Status string `json:"status"` + StoragePath string `json:"storagePath"` +} + type Service struct { - records repository.BackupRecordRepository - now func() time.Time + records repository.BackupRecordRepository + repositoryKey []byte + now func() time.Time } -func NewService(records repository.BackupRecordRepository) *Service { - return &Service{records: records, now: func() time.Time { return time.Now().UTC() }} +func NewService(records repository.BackupRecordRepository, repositoryKey ...[]byte) *Service { + var key []byte + if len(repositoryKey) > 0 { + key = append([]byte(nil), repositoryKey[0]...) + } + return &Service{records: records, repositoryKey: key, now: func() time.Time { return time.Now().UTC() }} } func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider storage.StorageProvider) (*CleanupResult, error) { + return s.cleanup(ctx, task, func(uint) (storage.StorageProvider, bool) { + return provider, provider != nil + }) +} + +// CleanupProviders applies one retention decision to every successful copy of +// a record before deleting its database row. This prevents multi-target tasks +// from leaving stale objects after the first target removes the shared record. +func (s *Service) CleanupProviders(ctx context.Context, task *model.BackupTask, providers map[uint]storage.StorageProvider) (*CleanupResult, error) { + return s.cleanup(ctx, task, func(targetID uint) (storage.StorageProvider, bool) { + provider, ok := providers[targetID] + return provider, ok && provider != nil + }) +} + +func (s *Service) cleanup(ctx context.Context, task *model.BackupTask, resolveProvider func(uint) (storage.StorageProvider, bool)) (*CleanupResult, error) { if task == nil { return nil, fmt.Errorf("backup task is required") } @@ -67,17 +101,35 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider // 差异链保护：保留仍被存活差异依赖的全量，避免删除基线后差异无法恢复。 candidates = protectDifferentialBases(records, candidates) result := &CleanupResult{} + repositoryProviders := make(map[uint]storage.StorageProvider) + touchedProviders := make(map[uint]storage.StorageProvider) for _, record := range candidates { - if strings.TrimSpace(record.StoragePath) != "" { - if provider == nil { - result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider for cleanup", record.ID)) + objects, objectErr := cleanupObjectsForRecord(record) + if objectErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("decode storage copies for record %d failed: %v", record.ID, objectErr)) + continue + } + allObjectsDeleted := true + for _, object := range objects { + provider, ok := resolveProvider(object.targetID) + if !ok { + result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider %d for cleanup", record.ID, object.targetID)) + allObjectsDeleted = false continue } - if err := provider.Delete(ctx, record.StoragePath); err != nil { - result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s failed: %v", record.StoragePath, err)) + if err := provider.Delete(ctx, object.path); err != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s from target %d failed: %v", object.path, object.targetID, err)) + allObjectsDeleted = false continue } result.DeletedObjects++ + touchedProviders[object.targetID] = provider + if record.BackupKind == model.BackupKindRepository { + repositoryProviders[object.targetID] = provider + } + } + if !allObjectsDeleted { + continue } if err := s.records.Delete(ctx, record.ID); err != nil { result.Warnings = append(result.Warnings, fmt.Sprintf("delete backup record %d failed: %v", record.ID, err)) @@ -85,13 +137,25 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider } result.DeletedRecords++ } + for targetID, provider := range repositoryProviders { + pruned, pruneErr := backup.NewRepositoryStore(s.repositoryKey).Prune(ctx, provider) + if pruneErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("prune CDC repository on target %d failed: %v", targetID, pruneErr)) + } else { + result.DeletedObjects += pruned.DeletedPacks + pruned.DeletedIndexes + } + } // 清理空目录：收集被删除文件的父目录，尝试移除空目录 - if dirCleaner, ok := provider.(storage.StorageDirCleaner); ok && result.DeletedObjects > 0 { + for targetID, provider := range touchedProviders { + dirCleaner, ok := provider.(storage.StorageDirCleaner) + if !ok { + continue + } prefixes := collectDirPrefixes(candidates) for _, prefix := range prefixes { if err := dirCleaner.RemoveEmptyDirs(ctx, prefix); err != nil { - result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s: %v", prefix, err)) + result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s on target %d: %v", prefix, targetID, err)) } } } @@ -99,6 +163,43 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider return result, nil } +func cleanupObjectsForRecord(record model.BackupRecord) ([]cleanupObject, error) { + defaultPath := strings.TrimSpace(record.StoragePath) + if strings.TrimSpace(record.StorageUploadResults) == "" { + if defaultPath == "" { + return nil, nil + } + return []cleanupObject{{targetID: record.StorageTargetID, path: defaultPath}}, nil + } + var results []storedUploadResult + if err := json.Unmarshal([]byte(record.StorageUploadResults), &results); err != nil { + return nil, err + } + objects := make([]cleanupObject, 0, len(results)) + seen := make(map[uint]struct{}, len(results)) + for _, result := range results { + if !strings.EqualFold(strings.TrimSpace(result.Status), model.BackupRecordStatusSuccess) { + continue + } + objectPath := strings.TrimSpace(result.StoragePath) + if objectPath == "" { + objectPath = defaultPath + } + if objectPath == "" { + continue + } + if _, exists := seen[result.StorageTargetID]; exists { + continue + } + seen[result.StorageTargetID] = struct{}{} + objects = append(objects, cleanupObject{targetID: result.StorageTargetID, path: objectPath}) + } + if len(objects) == 0 && defaultPath != "" { + return nil, fmt.Errorf("successful record has no successful storage copy") + } + return objects, nil +} + // protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」， // 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。 func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord { diff --git a/server/internal/backup/retention/service_test.go b/server/internal/backup/retention/service_test.go index a1e8124..8cb8108 100644 --- a/server/internal/backup/retention/service_test.go +++ b/server/internal/backup/retention/service_test.go @@ -221,3 +221,66 @@ func TestCleanupDeletesExpiredRecords(t *testing.T) { t.Fatalf("unexpected deleted objects: %#v", provider.deleted) } } + +func TestCleanupProvidersDeletesEverySuccessfulCopyBeforeRecord(t *testing.T) { + now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC) + completedNew := now.Add(-time.Hour) + completedOld := now.Add(-24 * time.Hour) + repo := &fakeRecordRepository{records: []model.BackupRecord{ + {ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew}, + { + ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld, + StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"},{"storageTargetId":13,"status":"failed"}]`, + }, + }} + first := &fakeProvider{} + second := &fakeProvider{} + service := NewService(repo) + service.now = func() time.Time { return now } + + result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{ + 11: first, + 12: second, + }) + if err != nil { + t.Fatalf("CleanupProviders returned error: %v", err) + } + if result.DeletedRecords != 1 || result.DeletedObjects != 2 || len(result.Warnings) != 0 { + t.Fatalf("unexpected cleanup result: %#v", result) + } + if len(repo.deleted) != 1 || repo.deleted[0] != 1 { + t.Fatalf("unexpected deleted records: %#v", repo.deleted) + } + if len(first.deleted) != 1 || first.deleted[0] != "first/1" { + t.Fatalf("unexpected first-target deletes: %#v", first.deleted) + } + if len(second.deleted) != 1 || second.deleted[0] != "second/1" { + t.Fatalf("unexpected second-target deletes: %#v", second.deleted) + } +} + +func TestCleanupProvidersKeepsRecordWhenCopyProviderIsUnavailable(t *testing.T) { + now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC) + completedNew := now.Add(-time.Hour) + completedOld := now.Add(-24 * time.Hour) + repo := &fakeRecordRepository{records: []model.BackupRecord{ + {ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew}, + { + ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld, + StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"}]`, + }, + }} + first := &fakeProvider{} + service := NewService(repo) + + result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{11: first}) + if err != nil { + t.Fatalf("CleanupProviders returned error: %v", err) + } + if result.DeletedRecords != 0 || result.DeletedObjects != 1 || len(result.Warnings) != 1 { + t.Fatalf("unexpected safe partial-cleanup result: %#v", result) + } + if len(repo.deleted) != 0 { + t.Fatalf("record must remain until all copies are deleted: %#v", repo.deleted) + } +} diff --git a/server/internal/backup/temp_files.go b/server/internal/backup/temp_files.go index db77fc2..cce68c5 100644 --- a/server/internal/backup/temp_files.go +++ b/server/internal/backup/temp_files.go @@ -55,6 +55,17 @@ func BuildStorageKey(backupType string, startedAt time.Time, fileName string) st return filepath.ToSlash(filepath.Join("BackupX", typeName, startedAt.UTC().Format("060102"), fileName)) } +// BuildRecordStorageKey gives remote-Agent artifacts a record-owned namespace. +// The Master validates this namespace before accepting a relayed upload, so one +// Agent cannot overwrite another record's object on centrally mounted storage. +func BuildRecordStorageKey(backupType string, startedAt time.Time, recordID uint, fileName string) string { + typeName := strings.TrimSpace(strings.ToLower(backupType)) + if typeName == "" { + typeName = "file" + } + return filepath.ToSlash(filepath.Join("BackupX", typeName, startedAt.UTC().Format("060102"), "records", fmt.Sprintf("%d", recordID), fileName)) +} + func sanitizeTaskName(value string) string { trimmed := strings.TrimSpace(strings.ToLower(value)) trimmed = strings.ReplaceAll(trimmed, " ", "-") diff --git a/server/internal/http/agent_handler.go b/server/internal/http/agent_handler.go index 8750c8f..23d28c1 100644 --- a/server/internal/http/agent_handler.go +++ b/server/internal/http/agent_handler.go @@ -156,6 +156,44 @@ func (h *AgentHandler) UpdateRecord(c *gin.Context) { response.Success(c, gin.H{"status": "ok"}) } +// UploadArtifact streams a remote source artifact into storage mounted only on +// the Master. The request body is never buffered as a whole in memory or disk. +func (h *AgentHandler) UploadArtifact(c *gin.Context) { + node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c)) + if err != nil { + response.Error(c, err) + return + } + recordID, err := strconv.ParseUint(c.Param("id"), 10, 32) + if err != nil { + response.Error(c, err) + return + } + targetID, err := strconv.ParseUint(c.Param("targetId"), 10, 32) + if err != nil { + response.Error(c, err) + return + } + if c.Request.ContentLength < 0 { + c.JSON(stdhttp.StatusLengthRequired, gin.H{"code": "CONTENT_LENGTH_REQUIRED", "message": "artifact content length is required"}) + return + } + if err := h.agentService.UploadArtifact( + c.Request.Context(), + node, + uint(recordID), + uint(targetID), + c.GetHeader("X-BackupX-Object-Key"), + c.Request.ContentLength, + c.GetHeader("X-BackupX-SHA256"), + c.Request.Body, + ); err != nil { + response.Error(c, err) + return + } + response.Success(c, gin.H{"status": "ok"}) +} + // GetRestoreSpec Agent 拉取恢复规格。 func (h *AgentHandler) GetRestoreSpec(c *gin.Context) { if h.restoreService == nil { @@ -208,6 +246,34 @@ func (h *AgentHandler) UpdateRestore(c *gin.Context) { response.Success(c, gin.H{"status": "ok"}) } +// DownloadRestoreArtifact streams a Master-local backup back to its source +// Agent for restore without exposing the local storage configuration. +func (h *AgentHandler) DownloadRestoreArtifact(c *gin.Context) { + if h.restoreService == nil { + c.JSON(stdhttp.StatusServiceUnavailable, gin.H{"code": "RESTORE_SERVICE_DISABLED", "message": "restore service is not enabled"}) + return + } + node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c)) + if err != nil { + response.Error(c, err) + return + } + restoreID, err := strconv.ParseUint(c.Param("id"), 10, 32) + if err != nil { + response.Error(c, err) + return + } + artifact, err := h.restoreService.DownloadAgentArtifact(c.Request.Context(), node, uint(restoreID)) + if err != nil { + response.Error(c, err) + return + } + c.DataFromReader(stdhttp.StatusOK, artifact.Size, "application/octet-stream", artifact.Reader, nil) + if err := artifact.Reader.Close(); err != nil { + _ = c.Error(err) + } +} + // Self 返回当前 Agent token 所属节点的状态，供安装脚本末尾探活。 func (h *AgentHandler) Self(c *gin.Context) { node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c)) diff --git a/server/internal/http/router.go b/server/internal/http/router.go index ba5bf9d..f003fb2 100644 --- a/server/internal/http/router.go +++ b/server/internal/http/router.go @@ -322,7 +322,9 @@ func NewRouter(deps RouterDependencies) *gin.Engine { agent.POST("/commands/:id/result", agentHandler.SubmitCommandResult) agent.GET("/tasks/:id", agentHandler.GetTaskSpec) agent.POST("/records/:id", agentHandler.UpdateRecord) + agent.PUT("/records/:id/artifacts/:targetId", agentHandler.UploadArtifact) agent.GET("/restores/:id/spec", agentHandler.GetRestoreSpec) + agent.GET("/restores/:id/artifact", agentHandler.DownloadRestoreArtifact) agent.POST("/restores/:id", agentHandler.UpdateRestore) // Agent v1（安装脚本探活用），仅 Self 端点 diff --git a/server/internal/installscript/deploy_install_test.go b/server/internal/installscript/deploy_install_test.go index 4c0507a..c001e23 100644 --- a/server/internal/installscript/deploy_install_test.go +++ b/server/internal/installscript/deploy_install_test.go @@ -39,3 +39,23 @@ func TestDeployInstallScriptSupportsReleasePackageLayout(t *testing.T) { } } } + +func TestDeployInstallScriptSupportsSourceBuildAndVerifiesFirstSetup(t *testing.T) { + scriptPath := filepath.Join("..", "..", "..", "deploy", "install.sh") + data, err := os.ReadFile(scriptPath) + if err != nil { + t.Fatal(err) + } + script := string(data) + for _, want := range []string{ + `SOURCE_BIN_DEFAULT="$PROJECT_ROOT/server/bin/backupx"`, + `For a source install, run 'make build' in the repository root first.`, + `HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8340/api/auth/setup/status}"`, + `systemctl is-active --quiet "$SERVICE_NAME"`, + `System setup`, + } { + if !strings.Contains(script, want) { + t.Fatalf("install.sh missing %q", want) + } + } +} diff --git a/server/internal/model/backup_record.go b/server/internal/model/backup_record.go index 9f8f9c6..8734c19 100644 --- a/server/internal/model/backup_record.go +++ b/server/internal/model/backup_record.go @@ -9,9 +9,11 @@ const ( ) const ( - // BackupKindFull 全量备份；BackupKindDifferential 差异备份（仅含自基线全量以来的变更）。 + // BackupKindFull 全量归档；BackupKindDifferential 差异归档； + // BackupKindRepository 为可独立恢复的 CDC 内容寻址快照。 BackupKindFull = "full" BackupKindDifferential = "differential" + BackupKindRepository = "repository" ) type BackupRecord struct { @@ -20,20 +22,22 @@ type BackupRecord struct { Task BackupTask `json:"task,omitempty"` StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"` StorageTarget StorageTarget `json:"storageTarget,omitempty"` - // NodeID 执行该次备份的节点（0 = 本机 Master）。用于集群中识别 local_disk 类型 - // 存储的归属节点，避免 Master 端试图跨节点访问远程 Agent 的本地存储。 - NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"` - Status string `gorm:"size:20;index;not null" json:"status"` - FileName string `gorm:"column:file_name;size:255" json:"fileName"` - FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"` - Checksum string `gorm:"column:checksum;size:64" json:"checksum"` - StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"` + // NodeID 执行该次备份的节点（0 = 本机 Master）。StorageTransferMode 进一步 + // 区分远程 Agent 直写与 Master 中转，避免在错误节点访问 local_disk。 + NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"` + Status string `gorm:"size:20;index;not null" json:"status"` + FileName string `gorm:"column:file_name;size:255" json:"fileName"` + FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"` + Checksum string `gorm:"column:checksum;size:64" json:"checksum"` + StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"` + // 空值表示旧版 Agent 直写；direct / master_relay 记录新协议的实际数据路径。 + StorageTransferMode string `gorm:"column:storage_transfer_mode;size:20" json:"storageTransferMode,omitempty"` StorageUploadResults string `gorm:"column:storage_upload_results;type:text" json:"-"` DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"` // Locked 保留锁定（法律保留）：为 true 时该备份不参与保留期/数量自动清理， // 且禁止手动删除，直到显式解锁。用于保护合规快照、迁移前基线等关键备份。 Locked bool `gorm:"column:locked;not null;default:false;index" json:"locked"` - // BackupKind 备份类型：full（全量）/ differential（差异）。 + // BackupKind 备份类型：full（全量）/ differential（差异）/ repository（CDC 快照）。 BackupKind string `gorm:"column:backup_kind;size:16;not null;default:'full';index" json:"backupKind"` // BaseRecordID 差异备份所基于的全量备份记录 ID（全量记录为 0）。 BaseRecordID uint `gorm:"column:base_record_id;index;not null;default:0" json:"baseRecordId"` diff --git a/server/internal/model/backup_task.go b/server/internal/model/backup_task.go index a3409d7..b077f68 100644 --- a/server/internal/model/backup_task.go +++ b/server/internal/model/backup_task.go @@ -12,9 +12,11 @@ const ( ) const ( - // BackupModeFull 全量模式（默认）；BackupModeDifferential 差异模式（仅文件类型本机任务）。 + // BackupModeFull 全量模式（默认）；BackupModeDifferential 差异归档； + // BackupModeRepository 为 CDC 内容寻址仓库模式（仅文件类型本机任务）。 BackupModeFull = "full" BackupModeDifferential = "differential" + BackupModeRepository = "repository" ) const ( @@ -55,7 +57,8 @@ type BackupTask struct { Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"` Encrypt bool `gorm:"not null;default:false" json:"encrypt"` MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"` - // BackupMode 备份模式：full（全量，默认）/ differential（差异）。差异仅支持本机文件任务。 + // BackupMode 备份模式：full（全量，默认）/ differential（差异归档）/ + // repository（FastCDC 分块、全局去重快照）。后两者仅支持本机文件任务。 BackupMode string `gorm:"column:backup_mode;size:16;not null;default:'full'" json:"backupMode"` // DiffFullIntervalDays 差异模式下强制全量的间隔（天）：最近全量超过该天数则本次自动改为全量， // 限制差异链跨度与单个差异体积。默认 7。 diff --git a/server/internal/service/agent_service.go b/server/internal/service/agent_service.go index 7723325..9a32e14 100644 --- a/server/internal/service/agent_service.go +++ b/server/internal/service/agent_service.go @@ -2,15 +2,22 @@ package service import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" + "io" + "math" + "path" "strings" "time" "backupx/server/internal/apperror" + "backupx/server/internal/backup" "backupx/server/internal/model" "backupx/server/internal/repository" + "backupx/server/internal/storage" "backupx/server/internal/storage/codec" ) @@ -23,6 +30,7 @@ type AgentService struct { storageRepo repository.StorageTargetRepository cmdRepo repository.AgentCommandRepository restoreRepo repository.RestoreRecordRepository + registry *storage.Registry cipher *codec.ConfigCipher } @@ -33,6 +41,7 @@ func NewAgentService( storageRepo repository.StorageTargetRepository, cmdRepo repository.AgentCommandRepository, cipher *codec.ConfigCipher, + registry *storage.Registry, ) *AgentService { return &AgentService{ nodeRepo: nodeRepo, @@ -40,6 +49,7 @@ func NewAgentService( recordRepo: recordRepo, storageRepo: storageRepo, cmdRepo: cmdRepo, + registry: registry, cipher: cipher, } } @@ -145,10 +155,11 @@ type AgentTaskSpec struct { // AgentStorageTargetConfig 存储目标配置（已解密） type AgentStorageTargetConfig struct { - ID uint `json:"id"` - Type string `json:"type"` - Name string `json:"name"` - Config json.RawMessage `json:"config"` + ID uint `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Config json.RawMessage `json:"config"` + TransferMode string `json:"transferMode"` } // GetTaskSpec 返回 Agent 执行任务所需的完整规格。 @@ -187,11 +198,22 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID if err != nil { return nil, fmt.Errorf("decrypt storage config: %w", err) } + transferMode := storage.TransferModeDirect + if strings.EqualFold(target.Type, storage.TypeLocalDisk) { + var localConfig storage.LocalDiskConfig + if err := json.Unmarshal(configRaw, &localConfig); err != nil { + return nil, fmt.Errorf("decode local disk config: %w", err) + } + if localConfig.MasterRelay { + transferMode = storage.TransferModeMasterRelay + } + } storageTargets = append(storageTargets, AgentStorageTargetConfig{ - ID: target.ID, - Type: target.Type, - Name: target.Name, - Config: json.RawMessage(configRaw), + ID: target.ID, + Type: target.Type, + Name: target.Name, + Config: json.RawMessage(configRaw), + TransferMode: transferMode, }) } return &AgentTaskSpec{ @@ -214,6 +236,102 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID }, nil } +// UploadArtifact receives a remote Agent artifact as a stream and writes it +// with a provider created on the Master. The first supported use is local_disk, +// whose configured path belongs to the Master rather than the source Agent. +func (s *AgentService) UploadArtifact(ctx context.Context, node *model.Node, recordID, targetID uint, objectKey string, size int64, checksum string, reader io.Reader) error { + if node == nil || reader == nil || s.registry == nil { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "中转上传参数不完整", nil) + } + record, err := s.recordRepo.FindByID(ctx, recordID) + if err != nil { + return err + } + if record == nil { + return apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "记录不存在", nil) + } + task, err := s.taskRepo.FindByID(ctx, record.TaskID) + if err != nil { + return err + } + if task == nil || !recordBelongsToNode(record, task, node.ID) { + return apperror.Unauthorized("BACKUP_RECORD_FORBIDDEN", "记录不属于当前节点", nil) + } + if isBackupRecordTerminal(record.Status) { + return apperror.BadRequest("BACKUP_RECORD_TERMINAL", "备份记录已结束，不能继续上传产物", nil) + } + allowedTarget := false + for _, configuredTargetID := range collectTargetIDs(task) { + if configuredTargetID == targetID { + allowedTarget = true + break + } + } + if !allowedTarget { + return apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil) + } + target, err := s.storageRepo.FindByID(ctx, targetID) + if err != nil { + return err + } + if target == nil || !strings.EqualFold(target.Type, storage.TypeLocalDisk) { + return apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "仅 Master 本地磁盘目标需要中转上传", nil) + } + configMap := map[string]any{} + if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil { + return fmt.Errorf("decrypt storage config: %w", err) + } + masterRelay, _ := configMap["masterRelay"].(bool) + if !masterRelay { + return apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "该本地磁盘目标配置为 Agent 直接写入", nil) + } + cleanKey, keyErr := s.validateArtifactKey(record, task, objectKey, true) + if keyErr != nil { + return keyErr + } + checksum = strings.TrimSpace(checksum) + checksumBytes, checksumErr := hex.DecodeString(checksum) + if size < 0 || size == math.MaxInt64 || checksumErr != nil || len(checksumBytes) != sha256.Size { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "中转上传需要有效的大小和 SHA-256", checksumErr) + } + if target.QuotaBytes > 0 { + usage, usageErr := s.recordRepo.StorageUsage(ctx) + if usageErr != nil { + return fmt.Errorf("read storage usage: %w", usageErr) + } + currentUsed := int64(0) + for _, item := range usage { + if item.StorageTargetID == targetID { + currentUsed = item.TotalSize + break + } + } + if currentUsed >= target.QuotaBytes || size > target.QuotaBytes-currentUsed { + return apperror.BadRequest("BACKUP_STORAGE_QUOTA_EXCEEDED", fmt.Sprintf("超出存储目标配额（当前 %d，新增 %d，配额 %d）", currentUsed, size, target.QuotaBytes), nil) + } + } + provider, err := s.registry.Create(ctx, target.Type, configMap) + if err != nil { + return fmt.Errorf("create master relay provider: %w", err) + } + limited := io.LimitReader(reader, size+1) + hashed := newHashingReader(limited) + metadata := map[string]string{ + "taskId": fmt.Sprintf("%d", task.ID), + "recordId": fmt.Sprintf("%d", record.ID), + "sourceNodeId": fmt.Sprintf("%d", node.ID), + "transferMode": storage.TransferModeMasterRelay, + } + if err := provider.Upload(ctx, cleanKey, hashed, size, metadata); err != nil { + return errors.Join(fmt.Errorf("relay artifact to master storage: %w", err), provider.Delete(ctx, cleanKey)) + } + if hashed.n != size || !strings.EqualFold(hashed.Sum(), checksum) { + deleteErr := provider.Delete(ctx, cleanKey) + return errors.Join(fmt.Errorf("relayed artifact integrity mismatch: received %d of %d bytes", hashed.n, size), deleteErr) + } + return nil +} + func (s *AgentService) ensureTaskSpecAccess(ctx context.Context, node *model.Node, task *model.BackupTask) error { if task.NodeID == node.ID { return nil @@ -236,6 +354,7 @@ type AgentRecordUpdate struct { Checksum string `json:"checksum,omitempty"` StoragePath string `json:"storagePath,omitempty"` StorageTargetID uint `json:"storageTargetId,omitempty"` + StorageTransferMode string `json:"storageTransferMode,omitempty"` StorageUploadResults []StorageUploadResultItem `json:"storageUploadResults,omitempty"` ErrorMessage string `json:"errorMessage,omitempty"` LogAppend string `json:"logAppend,omitempty"` // 增量日志，追加到 record.log_content @@ -260,6 +379,99 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor if isBackupRecordTerminal(record.Status) { return nil } + allowedTargets := make(map[uint]struct{}) + for _, targetID := range collectTargetIDs(task) { + allowedTargets[targetID] = struct{}{} + } + targetCache := make(map[uint]*model.StorageTarget) + validateTransferMode := func(targetID uint, transferMode string) (string, error) { + if _, ok := allowedTargets[targetID]; !ok { + return "", apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil) + } + target := targetCache[targetID] + if target == nil { + var findErr error + target, findErr = s.storageRepo.FindByID(ctx, targetID) + if findErr != nil { + return "", findErr + } + if target == nil { + return "", apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil) + } + targetCache[targetID] = target + } + expectedMode := storage.TransferModeDirect + if strings.EqualFold(target.Type, storage.TypeLocalDisk) { + var localConfig storage.LocalDiskConfig + if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &localConfig); err != nil { + return "", fmt.Errorf("decrypt storage config: %w", err) + } + if localConfig.MasterRelay { + expectedMode = storage.TransferModeMasterRelay + } + } + if transferMode != "" && transferMode != expectedMode { + return "", apperror.BadRequest("AGENT_STORAGE_TRANSFER_MODE_INVALID", "Agent 上报的存储传输模式与目标配置不一致", nil) + } + return expectedMode, nil + } + selectedTransferMode := "" + if update.StorageTargetID > 0 { + if _, ok := allowedTargets[update.StorageTargetID]; !ok { + return apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil) + } + var modeErr error + selectedTransferMode, modeErr = validateTransferMode(update.StorageTargetID, update.StorageTransferMode) + if modeErr != nil { + return modeErr + } + } else if update.StorageTransferMode != "" { + return apperror.BadRequest("AGENT_STORAGE_TRANSFER_MODE_INVALID", "传输模式缺少对应的存储目标", nil) + } + for index := range update.StorageUploadResults { + result := &update.StorageUploadResults[index] + expectedMode, err := validateTransferMode(result.StorageTargetID, result.TransferMode) + if err != nil { + return err + } + if result.FileSize < 0 || (result.Status != "" && result.Status != "success" && result.Status != "failed") { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的存储结果无效", nil) + } + if result.StoragePath != "" { + normalizedPath, err := s.validateArtifactKey(record, task, result.StoragePath, expectedMode == storage.TransferModeMasterRelay) + if err != nil { + return err + } + result.StoragePath = normalizedPath + } + result.TransferMode = expectedMode + } + if update.StoragePath != "" { + if update.StorageTargetID == 0 { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "存储路径缺少对应的存储目标", nil) + } + cleanStoragePath, err := s.validateArtifactKey(record, task, update.StoragePath, selectedTransferMode == storage.TransferModeMasterRelay) + if err != nil { + return err + } + if update.FileName != "" && path.Base(cleanStoragePath) != update.FileName { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "Agent 上报的文件名与存储路径不一致", nil) + } + update.StoragePath = cleanStoragePath + } + if update.Status != "" && update.Status != model.BackupRecordStatusRunning && update.Status != model.BackupRecordStatusSuccess && update.Status != model.BackupRecordStatusFailed { + return apperror.BadRequest("BACKUP_RECORD_STATUS_INVALID", "Agent 上报的备份状态无效", nil) + } + if update.FileSize < 0 || (update.FileName != "" && (path.Base(update.FileName) != update.FileName || strings.Contains(update.FileName, "\\"))) { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的备份文件信息无效", nil) + } + if update.Checksum != "" { + checksumBytes, checksumErr := hex.DecodeString(strings.TrimSpace(update.Checksum)) + if checksumErr != nil || len(checksumBytes) != sha256.Size { + return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的 SHA-256 无效", checksumErr) + } + update.Checksum = strings.ToLower(strings.TrimSpace(update.Checksum)) + } if update.Status != "" { record.Status = update.Status } @@ -277,6 +489,7 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor } if update.StorageTargetID > 0 { record.StorageTargetID = update.StorageTargetID + record.StorageTransferMode = selectedTransferMode } if len(update.StorageUploadResults) > 0 { if resultsJSON, marshalErr := json.Marshal(update.StorageUploadResults); marshalErr == nil { @@ -312,6 +525,27 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor return nil } +func (s *AgentService) validateArtifactKey(record *model.BackupRecord, task *model.BackupTask, objectKey string, requireRecordNamespace bool) (string, error) { + if record == nil || task == nil { + return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "无法确认中转对象归属", nil) + } + rawKey := objectKey + cleanKey := path.Clean(rawKey) + fileName := path.Base(cleanKey) + if rawKey == "" || strings.TrimSpace(rawKey) != rawKey || cleanKey == "." || path.IsAbs(cleanKey) || strings.HasPrefix(cleanKey, "../") || cleanKey != rawKey || strings.Contains(rawKey, "\\") || fileName == "." || fileName == "/" { + return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "中转上传对象路径不安全", nil) + } + expectedKey := backup.BuildRecordStorageKey(task.Type, record.StartedAt, record.ID, fileName) + if requireRecordNamespace { + legacyKey := backup.BuildStorageKey(task.Type, record.StartedAt, fileName) + if cleanKey != expectedKey && cleanKey != legacyKey { + return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "中转上传对象不属于当前备份记录", nil) + } + return expectedKey, nil + } + return cleanKey, nil +} + func recordBelongsToNode(record *model.BackupRecord, task *model.BackupTask, nodeID uint) bool { if record.NodeID != 0 { return record.NodeID == nodeID diff --git a/server/internal/service/agent_service_test.go b/server/internal/service/agent_service_test.go index ccda6f4..4a66dfd 100644 --- a/server/internal/service/agent_service_test.go +++ b/server/internal/service/agent_service_test.go @@ -1,19 +1,26 @@ package service import ( + "bytes" "context" + "crypto/sha256" "errors" + "fmt" + "os" "path/filepath" "strings" "testing" "time" + "backupx/server/internal/backup" "backupx/server/internal/config" "backupx/server/internal/database" "backupx/server/internal/logger" "backupx/server/internal/model" "backupx/server/internal/repository" + "backupx/server/internal/storage" "backupx/server/internal/storage/codec" + storageRclone "backupx/server/internal/storage/rclone" "gorm.io/gorm" ) @@ -42,7 +49,7 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo if err := nodeRepo.Create(context.Background(), other); err != nil { t.Fatalf("create other node: %v", err) } - targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()}) + targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir(), "masterRelay": true}) if err != nil { t.Fatalf("EncryptJSON returned error: %v", err) } @@ -76,7 +83,8 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo if err := recordRepo.Create(context.Background(), record); err != nil { t.Fatalf("create record: %v", err) } - return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher), db, recordRepo, cmdRepo, owner, other + storageRegistry := storage.NewRegistry(storageRclone.NewLocalDiskFactory()) + return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher, storageRegistry), db, recordRepo, cmdRepo, owner, other } func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) { @@ -90,19 +98,27 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing. if spec.TaskID != 1 || len(spec.StorageTargets) != 1 { t.Fatalf("unexpected spec: %#v", spec) } + if spec.StorageTargets[0].TransferMode != storage.TransferModeMasterRelay { + t.Fatalf("expected local disk to use Master relay, got %#v", spec.StorageTargets[0]) + } if _, err := svc.GetTaskSpec(ctx, other, 1); err == nil { t.Fatal("expected non-owner node to be forbidden from pooled task spec") } + record, err := records.FindByID(ctx, 1) + if err != nil { + t.Fatalf("FindByID record returned error: %v", err) + } + storagePath := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "backup.tar.gz") if err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{ - Status: model.BackupRecordStatusSuccess, - FileName: "backup.tar.gz", - FileSize: 123, - StoragePath: "tasks/1/backup.tar.gz", - StorageTargetID: 2, + Status: model.BackupRecordStatusSuccess, + FileName: "backup.tar.gz", + FileSize: 123, + StoragePath: storagePath, + StorageTargetID: 1, + StorageTransferMode: storage.TransferModeMasterRelay, StorageUploadResults: []StorageUploadResultItem{ - {StorageTargetID: 1, StorageTargetName: "first", Status: "failed", Error: "boom"}, - {StorageTargetID: 2, StorageTargetName: "second", Status: "success", StoragePath: "tasks/1/backup.tar.gz", FileSize: 123}, + {StorageTargetID: 1, StorageTargetName: "local", Status: "success", StoragePath: storagePath, FileSize: 123, TransferMode: storage.TransferModeMasterRelay}, }, }); err != nil { t.Fatalf("owner UpdateRecord returned error: %v", err) @@ -114,10 +130,13 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing. if updated.Status != model.BackupRecordStatusSuccess || updated.NodeID != owner.ID { t.Fatalf("unexpected updated record: %#v", updated) } - if updated.StorageTargetID != 2 { - t.Fatalf("expected successful storage target id 2, got %d", updated.StorageTargetID) + if updated.StorageTargetID != 1 { + t.Fatalf("expected successful storage target id 1, got %d", updated.StorageTargetID) + } + if updated.StorageTransferMode != storage.TransferModeMasterRelay { + t.Fatalf("expected Master relay transfer mode, got %q", updated.StorageTransferMode) } - if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"second"`) { + if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"local"`) { t.Fatalf("expected upload results to be persisted, got %q", updated.StorageUploadResults) } if err := svc.UpdateRecord(ctx, other, 1, AgentRecordUpdate{LogAppend: "bad"}); err == nil { @@ -125,6 +144,157 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing. } } +func TestAgentServiceRelaysRemoteArtifactToMasterLocalDisk(t *testing.T) { + svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t) + ctx := context.Background() + payload := []byte("artifact from remote source server") + digest := sha256.Sum256(payload) + checksum := fmt.Sprintf("%x", digest[:]) + record, err := records.FindByID(ctx, 1) + if err != nil { + t.Fatalf("FindByID record returned error: %v", err) + } + objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "remote-source.tar") + + if err := svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err != nil { + t.Fatalf("UploadArtifact returned error: %v", err) + } + target, err := svc.storageRepo.FindByID(ctx, 1) + if err != nil || target == nil { + t.Fatalf("FindByID target: target=%#v err=%v", target, err) + } + config := map[string]any{} + if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil { + t.Fatalf("DecryptJSON target config: %v", err) + } + basePath, _ := config["basePath"].(string) + stored, err := os.ReadFile(filepath.Join(basePath, filepath.FromSlash(objectKey))) + if err != nil { + t.Fatalf("read relayed artifact: %v", err) + } + if !bytes.Equal(stored, payload) { + t.Fatalf("relayed artifact differs: got %q", stored) + } + if err := svc.UploadArtifact(ctx, other, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err == nil { + t.Fatal("expected a different node to be forbidden from relaying the artifact") + } + + legacyPayload := []byte("artifact from an older Agent") + legacyDigest := sha256.Sum256(legacyPayload) + legacyKey := backup.BuildStorageKey("file", record.StartedAt, "legacy-agent.tar") + canonicalKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy-agent.tar") + if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, legacyKey, int64(len(legacyPayload)), fmt.Sprintf("%x", legacyDigest[:]), bytes.NewReader(legacyPayload)); err != nil { + t.Fatalf("UploadArtifact legacy key returned error: %v", err) + } + stored, err = os.ReadFile(filepath.Join(basePath, filepath.FromSlash(canonicalKey))) + if err != nil || !bytes.Equal(stored, legacyPayload) { + t.Fatalf("legacy Agent artifact was not normalized: data=%q err=%v", stored, err) + } + if _, err := os.Stat(filepath.Join(basePath, filepath.FromSlash(legacyKey))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("legacy object key should not be written directly: %v", err) + } + if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{ + Status: model.BackupRecordStatusSuccess, + FileName: "legacy-agent.tar", + FileSize: int64(len(legacyPayload)), + Checksum: fmt.Sprintf("%x", legacyDigest[:]), + StoragePath: legacyKey, + StorageTargetID: target.ID, + StorageUploadResults: []StorageUploadResultItem{{ + StorageTargetID: target.ID, + Status: "success", + StoragePath: legacyKey, + FileSize: int64(len(legacyPayload)), + }}, + }); err != nil { + t.Fatalf("UpdateRecord legacy key returned error: %v", err) + } + updated, err := records.FindByID(ctx, record.ID) + if err != nil { + t.Fatalf("FindByID updated record returned error: %v", err) + } + if updated.StoragePath != canonicalKey || updated.StorageTransferMode != storage.TransferModeMasterRelay || !strings.Contains(updated.StorageUploadResults, canonicalKey) { + t.Fatalf("legacy Agent record was not normalized: %#v", updated) + } +} + +func TestAgentServiceRejectsArtifactOutsideRecordNamespace(t *testing.T) { + svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t) + ctx := context.Background() + record, err := records.FindByID(ctx, 1) + if err != nil { + t.Fatalf("FindByID record returned error: %v", err) + } + target, err := svc.storageRepo.FindByID(ctx, 1) + if err != nil || target == nil { + t.Fatalf("FindByID target: target=%#v err=%v", target, err) + } + config := map[string]any{} + if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil { + t.Fatalf("DecryptJSON target config: %v", err) + } + basePath, _ := config["basePath"].(string) + victimKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID+1, "victim.tar") + victimPath := filepath.Join(basePath, filepath.FromSlash(victimKey)) + if err := os.MkdirAll(filepath.Dir(victimPath), 0o755); err != nil { + t.Fatalf("MkdirAll victim parent: %v", err) + } + if err := os.WriteFile(victimPath, []byte("keep me"), 0o600); err != nil { + t.Fatalf("WriteFile victim: %v", err) + } + payload := []byte("overwrite") + digest := sha256.Sum256(payload) + if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, victimKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload)); err == nil { + t.Fatal("expected another record namespace to be rejected") + } + stored, err := os.ReadFile(victimPath) + if err != nil { + t.Fatalf("ReadFile victim: %v", err) + } + if string(stored) != "keep me" { + t.Fatalf("victim object changed: %q", stored) + } + if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{StoragePath: victimKey, StorageTargetID: target.ID, StorageTransferMode: storage.TransferModeMasterRelay}); err == nil { + t.Fatal("expected another record namespace in status update to be rejected") + } +} + +func TestAgentServiceKeepsExistingLocalDiskTargetsAgentLocal(t *testing.T) { + svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t) + ctx := context.Background() + target, err := svc.storageRepo.FindByID(ctx, 1) + if err != nil || target == nil { + t.Fatalf("FindByID target: target=%#v err=%v", target, err) + } + legacyConfig, err := svc.cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()}) + if err != nil { + t.Fatalf("EncryptJSON legacy target: %v", err) + } + target.ConfigCiphertext = legacyConfig + if err := svc.storageRepo.Update(ctx, target); err != nil { + t.Fatalf("Update legacy target: %v", err) + } + + spec, err := svc.GetTaskSpec(ctx, owner, 1) + if err != nil { + t.Fatalf("GetTaskSpec returned error: %v", err) + } + if len(spec.StorageTargets) != 1 || spec.StorageTargets[0].TransferMode != storage.TransferModeDirect { + t.Fatalf("expected legacy local disk to stay Agent-local, got %#v", spec.StorageTargets) + } + payload := []byte("must not be relayed") + digest := sha256.Sum256(payload) + record, findErr := records.FindByID(ctx, 1) + if findErr != nil { + t.Fatalf("FindByID record returned error: %v", findErr) + } + objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy.tar") + err = svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload)) + if err == nil { + t.Fatal("expected relay upload to be rejected for an Agent-local target") + } +} + func TestAgentServiceUpdateRecordRefreshesTaskSummaryOnTerminalStatus(t *testing.T) { for _, status := range []string{model.BackupRecordStatusSuccess, model.BackupRecordStatusFailed} { t.Run(status, func(t *testing.T) { diff --git a/server/internal/service/backup_execution_service.go b/server/internal/service/backup_execution_service.go index 76066b2..f880682 100644 --- a/server/internal/service/backup_execution_service.go +++ b/server/internal/service/backup_execution_service.go @@ -5,11 +5,13 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "hash" "io" "os" "path/filepath" + "sort" "strings" "sync" "time" @@ -49,6 +51,7 @@ type StorageUploadResultItem struct { Status string `json:"status"` StoragePath string `json:"storagePath,omitempty"` FileSize int64 `json:"fileSize,omitempty"` + TransferMode string `json:"transferMode,omitempty"` Error string `json:"error,omitempty"` } @@ -62,6 +65,17 @@ type DownloadedArtifact struct { Reader io.ReadCloser } +type temporaryArtifactReader struct { + *os.File + directory string +} + +func (r *temporaryArtifactReader) Close() error { + closeErr := r.File.Close() + removeErr := os.RemoveAll(r.directory) + return errors.Join(closeErr, removeErr) +} + // collectTargetIDs 获取任务关联的所有存储目标 ID func collectTargetIDs(task *model.BackupTask) []uint { if len(task.StorageTargets) > 0 { @@ -102,6 +116,10 @@ type BackupExecutionService struct { bandwidthLimit string // rclone 带宽限制（全局默认，节点配置可覆盖） metrics *metrics.Metrics taskLocks sync.Map + // repositoryLocks serializes immutable index updates per storage target. + // Repository mode is intentionally single-writer in v1 to avoid orphaned + // duplicate packs when two local tasks discover the same missing chunk. + repositoryLocks sync.Map } // SetMetrics 注入 Prometheus 采集器。nil 时所有埋点退化为 no-op。 @@ -211,6 +229,25 @@ func (s *BackupExecutionService) DownloadRecord(ctx context.Context, recordID ui if err != nil { return nil, err } + if record.BackupKind == model.BackupKindRepository { + tempDir, err := os.MkdirTemp(s.tempDir, "repository-download-*") + if err != nil { + return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法创建 CDC 导出目录", err) + } + exportName := fmt.Sprintf("backupx-record-%d.tar", record.ID) + exportPath := filepath.Join(tempDir, exportName) + store := backup.NewRepositoryStore(s.cipher.Key()) + if err := store.ExportTar(ctx, provider, record.StoragePath, record.Checksum, exportPath); err != nil { + cleanupErr := os.RemoveAll(tempDir) + return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法从 CDC 仓库导出归档", errors.Join(err, cleanupErr)) + } + file, err := os.Open(exportPath) + if err != nil { + cleanupErr := os.RemoveAll(tempDir) + return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法打开 CDC 导出归档", errors.Join(err, cleanupErr)) + } + return &DownloadedArtifact{FileName: exportName, Reader: &temporaryArtifactReader{File: file, directory: tempDir}}, nil + } reader, err := provider.Download(ctx, record.StoragePath) if err != nil { return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法下载备份文件", err) @@ -234,6 +271,16 @@ func (s *BackupExecutionService) RestoreRecord(ctx context.Context, recordID uin if task == nil { return apperror.New(404, "BACKUP_TASK_NOT_FOUND", "关联的备份任务不存在，无法执行恢复", fmt.Errorf("backup task %d not found", record.TaskID)) } + if record.BackupKind == model.BackupKindRepository { + spec, specErr := s.buildTaskSpec(task, record.StartedAt) + if specErr != nil { + return specErr + } + if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, backup.NopLogWriter{}); err != nil { + return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "从 CDC 仓库恢复备份失败", err) + } + return nil + } tempDir, err := os.MkdirTemp("", "backupx-restore-*") if err != nil { return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法创建恢复目录", err) @@ -291,6 +338,58 @@ func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint fmt.Sprintf("该全量备份仍有 %d 个差异备份依赖它，删除会导致这些差异无法恢复。请先删除相关差异备份或等待其过期。", deps), nil) } } + if record.BackupKind == model.BackupKindRepository { + copies := []StorageUploadResultItem{{ + StorageTargetID: record.StorageTargetID, + Status: model.BackupRecordStatusSuccess, + StoragePath: record.StoragePath, + }} + if strings.TrimSpace(record.StorageUploadResults) != "" { + if err := json.Unmarshal([]byte(record.StorageUploadResults), &copies); err != nil { + return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法解析 CDC 仓库副本信息，已停止删除以避免遗留数据", err) + } + } + copyPaths := make(map[uint]string, len(copies)) + for _, copy := range copies { + if strings.EqualFold(copy.Status, model.BackupRecordStatusSuccess) && strings.TrimSpace(copy.StoragePath) != "" { + copyPaths[copy.StorageTargetID] = copy.StoragePath + } + } + targetIDs := make([]uint, 0, len(copyPaths)) + for targetID := range copyPaths { + targetIDs = append(targetIDs, targetID) + } + sort.Slice(targetIDs, func(i, j int) bool { return targetIDs[i] < targetIDs[j] }) + unlocks := make([]func(), 0, len(targetIDs)) + for _, targetID := range targetIDs { + unlocks = append(unlocks, s.acquireRepositoryLock(targetID)) + } + defer func() { + for index := len(unlocks) - 1; index >= 0; index-- { + unlocks[index]() + } + }() + providers := make(map[uint]storage.StorageProvider, len(targetIDs)) + for _, targetID := range targetIDs { + provider, resolveErr := s.resolveProvider(ctx, targetID) + if resolveErr != nil { + return resolveErr + } + if deleteErr := provider.Delete(ctx, copyPaths[targetID]); deleteErr != nil { + return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法删除 CDC 仓库快照", deleteErr) + } + providers[targetID] = provider + } + for targetID, provider := range providers { + if _, pruneErr := backup.NewRepositoryStore(s.cipher.Key()).Prune(ctx, provider); pruneErr != nil { + return apperror.Internal("BACKUP_REPOSITORY_PRUNE_FAILED", fmt.Sprintf("无法清理存储目标 %d 的 CDC 仓库；记录暂时保留以便重试", targetID), pruneErr) + } + } + if err := s.records.Delete(ctx, recordID); err != nil { + return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法删除备份记录", err) + } + return nil + } if remote, err := s.deleteRemoteLocalDiskObject(ctx, record); err != nil { return err } else if !remote && strings.TrimSpace(record.StoragePath) != "" { @@ -312,6 +411,9 @@ func (s *BackupExecutionService) deleteRemoteLocalDiskObject(ctx context.Context if strings.TrimSpace(record.StoragePath) == "" || s.nodeRepo == nil { return false, nil } + if record.StorageTransferMode == storage.TransferModeMasterRelay { + return false, nil + } node, err := s.nodeRepo.FindByID(ctx, record.NodeID) if err != nil || node == nil || node.IsLocal { return false, nil @@ -384,6 +486,9 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b return nil, perr } } + if strings.EqualFold(task.BackupMode, model.BackupModeRepository) && s.resolveRemoteNode(ctx, resolvedNodeID) != nil { + return nil, apperror.BadRequest("BACKUP_TASK_REPOSITORY_REMOTE_UNSUPPORTED", "CDC 仓库模式当前仅支持 Master 本机单写者执行", nil) + } startedAt := s.now() // 取第一个存储目标 ID 做兼容 primaryTargetID := task.StorageTargetID @@ -630,6 +735,141 @@ func (s *BackupExecutionService) resolveDifferentialBase(ctx context.Context, ta return 0, backup.Manifest{}, false } +type repositoryTaskResult struct { + fileName string + logicalSize int64 + checksum string + storagePath string + storageTargetID uint + manifestJSON string + uploadResults []StorageUploadResultItem + providers map[uint]storage.StorageProvider +} + +func (s *BackupExecutionService) executeRepositoryTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, spec backup.TaskSpec, logger *backup.ExecutionLogger) (*repositoryTaskResult, error) { + store := backup.NewRepositoryStore(s.cipher.Key()) + plan, err := store.BuildPlan(ctx, spec, logger) + if err != nil { + return nil, err + } + defer func() { + if closeErr := plan.Close(); closeErr != nil { + logger.Warnf("清理 CDC 临时计划失败：%v", closeErr) + } + }() + manifestBytes, err := backup.EncodeManifest(plan.Manifest) + if err != nil { + return nil, fmt.Errorf("encode repository manifest: %w", err) + } + targetIDs := collectTargetIDs(task) + if len(targetIDs) == 0 { + return nil, fmt.Errorf("没有关联的存储目标") + } + storageUsage, usageErr := s.storageUsageSnapshot(ctx) + if usageErr != nil { + logger.Warnf("读取存储目标用量失败，跳过本次软配额校验：%v", usageErr) + storageUsage = map[uint]int64{} + } + + snapshotKey := store.SnapshotKey(task.ID, recordID, startedAt) + result := &repositoryTaskResult{ + fileName: filepath.Base(snapshotKey), + logicalSize: plan.LogicalSize, + storagePath: snapshotKey, + manifestJSON: string(manifestBytes), + uploadResults: make([]StorageUploadResultItem, 0, len(targetIDs)), + providers: make(map[uint]storage.StorageProvider), + } + var failures []string + for _, targetID := range targetIDs { + target, findErr := s.targets.FindByID(ctx, targetID) + targetName := fmt.Sprintf("target-%d", targetID) + if findErr == nil && target != nil { + targetName = target.Name + } + if findErr != nil || target == nil { + message := "存储目标不存在" + if findErr != nil { + message = findErr.Error() + } + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: message}) + failures = append(failures, fmt.Sprintf("%s: %s", targetName, message)) + continue + } + provider, resolveErr := s.resolveProviderForNode(ctx, targetID, task.NodeID) + if resolveErr != nil { + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: resolveErr.Error()}) + failures = append(failures, fmt.Sprintf("%s: %v", targetName, resolveErr)) + continue + } + logger.Infof("同步 CDC 仓库到存储目标：%s", targetName) + unlock := s.acquireRepositoryLock(targetID) + estimatedSize, estimateErr := store.EstimateUploadSize(ctx, provider, plan) + if estimateErr != nil { + unlock() + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: estimateErr.Error()}) + failures = append(failures, fmt.Sprintf("%s: %v", targetName, estimateErr)) + continue + } + if target.QuotaBytes > 0 && storageUsage[targetID]+estimatedSize > target.QuotaBytes { + unlock() + message := fmt.Sprintf("超出存储目标配额（%d + 预计 %d > %d）", storageUsage[targetID], estimatedSize, target.QuotaBytes) + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: message}) + failures = append(failures, fmt.Sprintf("%s: %s", targetName, message)) + continue + } + upload, uploadErr := store.Upload(ctx, provider, plan, snapshotKey) + unlock() + if uploadErr != nil { + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()}) + failures = append(failures, fmt.Sprintf("%s: %v", targetName, uploadErr)) + logger.Warnf("存储目标 %s CDC 仓库同步失败：%v", targetName, uploadErr) + continue + } + result.uploadResults = append(result.uploadResults, StorageUploadResultItem{ + StorageTargetID: targetID, StorageTargetName: targetName, Status: "success", + StoragePath: upload.SnapshotKey, FileSize: upload.UploadedBytes, + }) + result.providers[targetID] = provider + if result.storageTargetID == 0 { + result.storageTargetID = targetID + result.checksum = upload.Checksum + } + logger.Infof("存储目标 %s CDC 同步完成：新块 %d/%d，复用 %d bytes，实际上传 %d bytes", targetName, upload.NewChunks, upload.UniqueChunks, upload.ReusedBytes, upload.UploadedBytes) + } + if result.storageTargetID == 0 { + return nil, fmt.Errorf("所有存储目标 CDC 仓库同步均失败：%s", strings.Join(failures, "; ")) + } + if len(failures) > 0 { + logger.Warnf("部分存储目标 CDC 仓库同步失败：%s", strings.Join(failures, "; ")) + } + if s.dependentsResolver != nil { + go func(upstreamID uint, upstreamName string) { + dependents, resolveErr := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID) + if resolveErr != nil { + logger.Warnf("解析任务 %s 的下游依赖失败：%v", upstreamName, resolveErr) + return + } + for _, dependentID := range dependents { + if _, runErr := s.RunTaskByID(context.Background(), dependentID); runErr != nil { + logger.Warnf("触发下游任务 #%d 失败（上游: %s）：%v", dependentID, upstreamName, runErr) + } else { + logger.Infof("已触发下游任务 #%d（上游: %s）", dependentID, upstreamName) + } + } + }(task.ID, task.Name) + } + return result, nil +} + +func (s *BackupExecutionService) acquireRepositoryLock(targetID uint) func() { + created := &sync.Mutex{} + actual, _ := s.repositoryLocks.LoadOrStore(targetID, created) + lock := actual.(*sync.Mutex) + lock.Lock() + return lock.Unlock +} + func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) { // 节点级并发限流：当任务绑定节点且节点配置了 MaxConcurrent>0， // 该节点上所有任务共享一个节点专属 semaphore，互相排队 @@ -658,18 +898,33 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba backupKind := model.BackupKindFull var baseRecordID uint var manifestJSON string + var repositoryProviders map[uint]storage.StorageProvider completeRecord := func() { + readyForRepositoryRetention := status == model.BackupRecordStatusSuccess if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil { logger.Errorf("写回备份记录失败：%v", finalizeErr) + readyForRepositoryRetention = false } // 采集任务执行结果到 Prometheus（耗时 + 产出字节 + 状态计数） s.metrics.ObserveTaskRun(task.Type, status, time.Since(startedAt).Seconds(), fileSize) // 写入多目标上传结果 if len(uploadResults) > 0 { - if resultsJSON, marshalErr := json.Marshal(uploadResults); marshalErr == nil { - if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil { - record.StorageUploadResults = string(resultsJSON) - _ = s.records.Update(ctx, record) + resultsJSON, marshalErr := json.Marshal(uploadResults) + if marshalErr != nil { + logger.Warnf("序列化多目标上传结果失败：%v", marshalErr) + readyForRepositoryRetention = false + } else if record, findErr := s.records.FindByID(ctx, recordID); findErr != nil || record == nil { + if findErr != nil { + logger.Warnf("读取备份记录以写回多目标结果失败：%v", findErr) + } else { + logger.Warnf("备份记录 #%d 不存在，无法写回多目标结果", recordID) + } + readyForRepositoryRetention = false + } else { + record.StorageUploadResults = string(resultsJSON) + if updateErr := s.records.Update(ctx, record); updateErr != nil { + logger.Warnf("写回多目标上传结果失败：%v", updateErr) + readyForRepositoryRetention = false } } } @@ -681,6 +936,36 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba record.Manifest = manifestJSON if updErr := s.records.Update(ctx, record); updErr != nil { logger.Warnf("写回差异链信息失败：%v", updErr) + readyForRepositoryRetention = false + } + } else { + if findErr != nil { + logger.Warnf("读取备份记录以写回备份类型失败：%v", findErr) + } else { + logger.Warnf("备份记录 #%d 不存在，无法写回备份类型", recordID) + } + readyForRepositoryRetention = false + } + } + if readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 { + targetIDs := make([]uint, 0, len(repositoryProviders)) + for targetID := range repositoryProviders { + targetIDs = append(targetIDs, targetID) + } + sort.Slice(targetIDs, func(i, j int) bool { return targetIDs[i] < targetIDs[j] }) + unlocks := make([]func(), 0, len(targetIDs)) + for _, targetID := range targetIDs { + unlocks = append(unlocks, s.acquireRepositoryLock(targetID)) + } + cleanupResult, cleanupErr := s.retention.CleanupProviders(ctx, task, repositoryProviders) + for index := len(unlocks) - 1; index >= 0; index-- { + unlocks[index]() + } + if cleanupErr != nil { + logger.Warnf("执行 CDC 仓库保留策略失败：%v", cleanupErr) + } else { + for _, warning := range cleanupResult.Warnings { + logger.Warnf("CDC 仓库保留策略警告：%s", warning) } } } @@ -701,6 +986,26 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba logger.Errorf("构建任务运行时配置失败：%v", err) return } + if task.Type == model.BackupTaskTypeFile && strings.EqualFold(task.BackupMode, model.BackupModeRepository) { + backupKind = model.BackupKindRepository + repositoryResult, repositoryErr := s.executeRepositoryTask(ctx, task, recordID, startedAt, spec, logger) + if repositoryErr != nil { + errMessage = repositoryErr.Error() + logger.Errorf("执行 CDC 仓库备份失败：%v", repositoryErr) + return + } + fileName = repositoryResult.fileName + fileSize = repositoryResult.logicalSize + checksum = repositoryResult.checksum + storagePath = repositoryResult.storagePath + selectedStorageTargetID = repositoryResult.storageTargetID + uploadResults = repositoryResult.uploadResults + repositoryProviders = repositoryResult.providers + manifestJSON = repositoryResult.manifestJSON + status = model.BackupRecordStatusSuccess + logger.Infof("CDC 仓库备份执行完成") + return + } // 差异备份：解析基线全量，命中则切换为差异模式（仅本机文件任务）。 if baseID, baseManifest, ok := s.resolveDifferentialBase(ctx, task); ok { spec.Differential = true diff --git a/server/internal/service/backup_execution_service_test.go b/server/internal/service/backup_execution_service_test.go index 5f9ec15..f9eb3b5 100644 --- a/server/internal/service/backup_execution_service_test.go +++ b/server/internal/service/backup_execution_service_test.go @@ -1,6 +1,7 @@ package service import ( + "bytes" "context" "errors" "fmt" @@ -155,6 +156,108 @@ func TestBackupExecutionServiceRunTaskByIDSync(t *testing.T) { } } +func TestBackupExecutionServiceRepositoryModeRoundTrip(t *testing.T) { + executionService, recordService, tasks, _, records, sourceDir, storageDir := newExecutionTestServices(t) + ctx := context.Background() + task, err := tasks.FindByID(ctx, 1) + if err != nil { + t.Fatalf("FindByID task returned error: %v", err) + } + task.BackupMode = model.BackupModeRepository + task.Compression = "zstd" + if err := tasks.Update(ctx, task); err != nil { + t.Fatalf("Update repository task returned error: %v", err) + } + large := make([]byte, 4<<20) + for index := range large { + large[index] = byte((index * 31) % 251) + } + largePath := filepath.Join(sourceDir, "large.bin") + if err := os.WriteFile(largePath, large, 0o640); err != nil { + t.Fatalf("write large fixture: %v", err) + } + + first, err := executionService.RunTaskByIDSync(ctx, task.ID) + if err != nil { + t.Fatalf("first repository backup returned error: %v", err) + } + if first.Status != model.BackupRecordStatusSuccess || first.BackupKind != model.BackupKindRepository { + t.Fatalf("unexpected first repository record: %#v", first) + } + if !strings.HasPrefix(first.StoragePath, ".backupx/repository/v1/snapshots/") { + t.Fatalf("unexpected repository snapshot path: %s", first.StoragePath) + } + + large[2<<20] ^= 0xff + if err := os.WriteFile(largePath, large, 0o640); err != nil { + t.Fatalf("modify large fixture: %v", err) + } + second, err := executionService.RunTaskByIDSync(ctx, task.ID) + if err != nil { + t.Fatalf("second repository backup returned error: %v", err) + } + stored, err := records.FindByID(ctx, second.ID) + if err != nil { + t.Fatalf("FindByID repository record returned error: %v", err) + } + if stored == nil || stored.BackupKind != model.BackupKindRepository || stored.Manifest == "" { + t.Fatalf("repository metadata was not persisted: %#v", stored) + } + + download, err := recordService.Download(ctx, second.ID) + if err != nil { + t.Fatalf("export repository snapshot returned error: %v", err) + } + exported, readErr := io.ReadAll(download.Reader) + closeErr := download.Reader.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("read repository export: read=%v close=%v", readErr, closeErr) + } + if len(exported) == 0 || !strings.HasSuffix(download.FileName, ".tar") { + t.Fatalf("unexpected repository export: name=%s size=%d", download.FileName, len(exported)) + } + + if err := os.WriteFile(largePath, bytes.Repeat([]byte{0}, len(large)), 0o640); err != nil { + t.Fatalf("damage source before restore: %v", err) + } + if err := executionService.RestoreRecord(ctx, second.ID); err != nil { + t.Fatalf("restore repository record returned error: %v", err) + } + restored, err := os.ReadFile(largePath) + if err != nil { + t.Fatalf("read restored source: %v", err) + } + if !bytes.Equal(restored, large) { + t.Fatalf("repository restore did not reproduce the source") + } + + if err := recordService.Delete(ctx, first.ID); err != nil { + t.Fatalf("delete first repository record: %v", err) + } + if err := recordService.Delete(ctx, second.ID); err != nil { + t.Fatalf("delete second repository record: %v", err) + } + packRoot := filepath.Join(storageDir, filepath.FromSlash(".backupx/repository/v1/packs")) + remainingPacks := 0 + if err := filepath.Walk(packRoot, func(_ string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + if os.IsNotExist(walkErr) { + return nil + } + return walkErr + } + if info != nil && !info.IsDir() { + remainingPacks++ + } + return nil + }); err != nil { + t.Fatalf("inspect repository packs: %v", err) + } + if remainingPacks != 0 { + t.Fatalf("repository prune left %d packs after deleting all snapshots", remainingPacks) + } +} + func TestBackupExecutionServiceNodePoolSelectionDoesNotPersistTaskNodeID(t *testing.T) { executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t) ctx := context.Background() @@ -326,6 +429,56 @@ func TestBackupExecutionServiceRestoreRecordRejectsRemoteLocalDisk(t *testing.T) } } +func TestBackupExecutionServiceDownloadsMasterRelayedLocalDiskRecord(t *testing.T) { + executionService, _, tasks, _, records, _, storageDir := newExecutionTestServices(t) + ctx := context.Background() + executionService.SetClusterDependencies(&nodeRepoStub{nodes: []model.Node{ + {ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline}, + }}, &fakeDispatcher{}) + task, err := tasks.FindByID(ctx, 1) + if err != nil { + t.Fatalf("FindByID task returned error: %v", err) + } + storagePath := "file/2026/05/09/relayed.tar" + artifactPath := filepath.Join(storageDir, filepath.FromSlash(storagePath)) + if err := os.MkdirAll(filepath.Dir(artifactPath), 0o755); err != nil { + t.Fatalf("MkdirAll artifact parent returned error: %v", err) + } + content := []byte("stored on Master") + if err := os.WriteFile(artifactPath, content, 0o600); err != nil { + t.Fatalf("WriteFile artifact returned error: %v", err) + } + completedAt := time.Now().UTC() + record := &model.BackupRecord{ + TaskID: task.ID, + StorageTargetID: task.StorageTargetID, + NodeID: 10, + Status: model.BackupRecordStatusSuccess, + FileName: "relayed.tar", + FileSize: int64(len(content)), + StoragePath: storagePath, + StorageTransferMode: storage.TransferModeMasterRelay, + StartedAt: completedAt.Add(-time.Second), + CompletedAt: &completedAt, + } + if err := records.Create(ctx, record); err != nil { + t.Fatalf("Create record returned error: %v", err) + } + + download, err := executionService.DownloadRecord(ctx, record.ID) + if err != nil { + t.Fatalf("DownloadRecord returned error: %v", err) + } + got, readErr := io.ReadAll(download.Reader) + closeErr := download.Reader.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("read relayed artifact: read=%v close=%v", readErr, closeErr) + } + if !bytes.Equal(got, content) { + t.Fatalf("downloaded content = %q, want %q", got, content) + } +} + func TestBackupExecutionServiceRecordsFirstSuccessfulStorageTarget(t *testing.T) { executionService, _, tasks, targets, records, _, _ := newExecutionTestServices(t) ctx := context.Background() diff --git a/server/internal/service/backup_record_service.go b/server/internal/service/backup_record_service.go index e4d7246..3b2461a 100644 --- a/server/internal/service/backup_record_service.go +++ b/server/internal/service/backup_record_service.go @@ -23,22 +23,23 @@ type BackupRecordListInput struct { } type BackupRecordSummary struct { - ID uint `json:"id"` - TaskID uint `json:"taskId"` - TaskName string `json:"taskName"` - StorageTargetID uint `json:"storageTargetId"` - StorageTargetName string `json:"storageTargetName"` - Status string `json:"status"` - FileName string `json:"fileName"` - FileSize int64 `json:"fileSize"` - Checksum string `json:"checksum"` - StoragePath string `json:"storagePath"` - DurationSeconds int `json:"durationSeconds"` - ErrorMessage string `json:"errorMessage"` - StartedAt time.Time `json:"startedAt"` - CompletedAt *time.Time `json:"completedAt,omitempty"` - Locked bool `json:"locked"` - BackupKind string `json:"backupKind"` + ID uint `json:"id"` + TaskID uint `json:"taskId"` + TaskName string `json:"taskName"` + StorageTargetID uint `json:"storageTargetId"` + StorageTargetName string `json:"storageTargetName"` + Status string `json:"status"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + Checksum string `json:"checksum"` + StoragePath string `json:"storagePath"` + StorageTransferMode string `json:"storageTransferMode,omitempty"` + DurationSeconds int `json:"durationSeconds"` + ErrorMessage string `json:"errorMessage"` + StartedAt time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt,omitempty"` + Locked bool `json:"locked"` + BackupKind string `json:"backupKind"` } type BackupRecordDetail struct { @@ -184,22 +185,23 @@ func (s *BackupRecordService) SetLock(ctx context.Context, id uint, locked bool) func toBackupRecordSummary(item *model.BackupRecord) BackupRecordSummary { return BackupRecordSummary{ - ID: item.ID, - TaskID: item.TaskID, - TaskName: item.Task.Name, - StorageTargetID: item.StorageTargetID, - StorageTargetName: item.StorageTarget.Name, - Status: item.Status, - FileName: item.FileName, - FileSize: item.FileSize, - Checksum: item.Checksum, - StoragePath: item.StoragePath, - DurationSeconds: item.DurationSeconds, - ErrorMessage: item.ErrorMessage, - StartedAt: item.StartedAt, - CompletedAt: item.CompletedAt, - Locked: item.Locked, - BackupKind: item.BackupKind, + ID: item.ID, + TaskID: item.TaskID, + TaskName: item.Task.Name, + StorageTargetID: item.StorageTargetID, + StorageTargetName: item.StorageTarget.Name, + Status: item.Status, + FileName: item.FileName, + FileSize: item.FileSize, + Checksum: item.Checksum, + StoragePath: item.StoragePath, + StorageTransferMode: item.StorageTransferMode, + DurationSeconds: item.DurationSeconds, + ErrorMessage: item.ErrorMessage, + StartedAt: item.StartedAt, + CompletedAt: item.CompletedAt, + Locked: item.Locked, + BackupKind: item.BackupKind, } } diff --git a/server/internal/service/backup_task_service.go b/server/internal/service/backup_task_service.go index 998f327..87eee42 100644 --- a/server/internal/service/backup_task_service.go +++ b/server/internal/service/backup_task_service.go @@ -57,8 +57,8 @@ type BackupTaskUpsertInput struct { KeepWeekly int `json:"keepWeekly"` KeepMonthly int `json:"keepMonthly"` KeepYearly int `json:"keepYearly"` - // BackupMode 备份模式：full（默认）/ differential（差异，仅文件类型本机任务） - BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential"` + // BackupMode 备份模式：full（默认）/ differential（差异归档）/ repository（CDC 去重仓库） + BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential repository"` DiffFullIntervalDays int `json:"diffFullIntervalDays"` // 备份复制目标存储 ID 列表（3-2-1 规则） ReplicationTargetIDs []uint `json:"replicationTargetIds"` @@ -414,21 +414,50 @@ func (s *BackupTaskService) cleanupRemoteFiles(ctx context.Context, taskID uint) recordCount = len(records) // 缓存 provider 避免同一存储目标重复创建连接 providerCache := make(map[uint]storage.StorageProvider) + repositoryProviders := make(map[uint]storage.StorageProvider) for _, record := range records { - if strings.TrimSpace(record.StoragePath) == "" { - continue + copies := []StorageUploadResultItem{{ + StorageTargetID: record.StorageTargetID, + Status: model.BackupRecordStatusSuccess, + StoragePath: record.StoragePath, + }} + if strings.TrimSpace(record.StorageUploadResults) != "" { + var storedCopies []StorageUploadResultItem + if unmarshalErr := json.Unmarshal([]byte(record.StorageUploadResults), &storedCopies); unmarshalErr == nil { + copies = storedCopies + } } - provider, ok := providerCache[record.StorageTargetID] - if !ok { - provider, err = s.resolveStorageProvider(ctx, record.StorageTargetID) - if err != nil { + seenTargets := make(map[uint]struct{}, len(copies)) + for _, copy := range copies { + if !strings.EqualFold(copy.Status, model.BackupRecordStatusSuccess) || strings.TrimSpace(copy.StoragePath) == "" { + continue + } + if _, seen := seenTargets[copy.StorageTargetID]; seen { continue } - providerCache[record.StorageTargetID] = provider + seenTargets[copy.StorageTargetID] = struct{}{} + provider, ok := providerCache[copy.StorageTargetID] + if !ok { + provider, err = s.resolveStorageProvider(ctx, copy.StorageTargetID) + if err != nil { + continue + } + providerCache[copy.StorageTargetID] = provider + } + if err := provider.Delete(ctx, copy.StoragePath); err == nil { + cleanedFiles++ + if record.BackupKind == model.BackupKindRepository { + repositoryProviders[copy.StorageTargetID] = provider + } + } } - if err := provider.Delete(ctx, record.StoragePath); err == nil { - cleanedFiles++ + } + for _, provider := range repositoryProviders { + pruned, pruneErr := backup.NewRepositoryStore(s.cipher.Key()).Prune(ctx, provider) + if pruneErr != nil { + continue } + cleanedFiles += pruned.DeletedIndexes + pruned.DeletedPacks } return recordCount, cleanedFiles } @@ -530,6 +559,17 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B return apperror.BadRequest("BACKUP_TASK_DIFF_REMOTE_UNSUPPORTED", "差异备份当前仅支持本机 Master 执行，请将任务固定在本机或改用全量备份。", nil) } } + if strings.EqualFold(strings.TrimSpace(input.BackupMode), model.BackupModeRepository) { + if input.Type != model.BackupTaskTypeFile { + return apperror.BadRequest("BACKUP_TASK_REPOSITORY_UNSUPPORTED", "CDC 仓库模式仅支持文件目录类型任务", nil) + } + if strings.TrimSpace(input.NodePoolTag) != "" || (fixedNode != nil && !fixedNode.IsLocal) { + return apperror.BadRequest("BACKUP_TASK_REPOSITORY_REMOTE_UNSUPPORTED", "CDC 仓库模式当前采用单写者索引，仅支持 Master 本机执行。远程服务器备份请暂用全量模式。", nil) + } + if len(input.ReplicationTargetIDs) > 0 { + return apperror.BadRequest("BACKUP_TASK_REPOSITORY_REPLICATION_UNSUPPORTED", "CDC 仓库快照不能使用对象级复制；请直接为任务选择多个存储目标以生成完整仓库副本。", nil) + } + } if input.RetentionDays < 0 { return apperror.BadRequest("BACKUP_TASK_INVALID", "保留天数不能小于 0", nil) } @@ -935,10 +975,17 @@ func decodeExtraConfig(value string) (map[string]any, error) { return result, nil } -// normalizeBackupMode 归一化备份模式：仅文件类型可启用差异，其余一律全量（双保险，防绕过校验）。 +// normalizeBackupMode 归一化备份模式：仅文件类型可启用差异或 CDC 仓库， +// 其余一律全量（双保险，防绕过校验）。 func normalizeBackupMode(mode, taskType string) string { - if strings.EqualFold(strings.TrimSpace(mode), model.BackupModeDifferential) && normalizeBackupTaskType(taskType) == model.BackupTaskTypeFile { + if normalizeBackupTaskType(taskType) != model.BackupTaskTypeFile { + return model.BackupModeFull + } + switch strings.ToLower(strings.TrimSpace(mode)) { + case model.BackupModeDifferential: return model.BackupModeDifferential + case model.BackupModeRepository: + return model.BackupModeRepository } return model.BackupModeFull } diff --git a/server/internal/service/execution_helpers.go b/server/internal/service/execution_helpers.go index 4c04de5..2042a2c 100644 --- a/server/internal/service/execution_helpers.go +++ b/server/internal/service/execution_helpers.go @@ -140,6 +140,11 @@ func validateCrossNodeLocalDisk(ctx context.Context, nodeRepo repository.NodeRep if record == nil || record.NodeID == 0 || nodeRepo == nil { return nil } + // 中转模式的对象实际落在 Master 配置的本地磁盘，Master 可以安全访问。 + // 空值和 direct 均按旧版 Agent 本地落盘处理，保持升级兼容。 + if record.StorageTransferMode == storage.TransferModeMasterRelay { + return nil + } node, err := nodeRepo.FindByID(ctx, record.NodeID) if err != nil || node == nil || node.IsLocal { return nil diff --git a/server/internal/service/restore_service.go b/server/internal/service/restore_service.go index ba1c0db..5d67531 100644 --- a/server/internal/service/restore_service.go +++ b/server/internal/service/restore_service.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -321,6 +322,13 @@ func (s *RestoreService) restoreArtifact(ctx context.Context, record *model.Back if err != nil { return fmt.Errorf("创建存储客户端失败：%w", err) } + if record.BackupKind == model.BackupKindRepository { + logger.Infof("读取 CDC 仓库快照：%s", record.StoragePath) + if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, logger); err != nil { + return fmt.Errorf("恢复 CDC 仓库快照失败：%w", err) + } + return nil + } recDir, err := os.MkdirTemp(parentTempDir, fmt.Sprintf("rec-%d-*", record.ID)) if err != nil { return fmt.Errorf("创建恢复子目录失败：%w", err) @@ -368,6 +376,9 @@ func (s *RestoreService) buildRestoreChain(ctx context.Context, record *model.Ba } func backupKindLabel(kind string) string { + if kind == model.BackupKindRepository { + return "CDC 仓库快照" + } if kind == model.BackupKindDifferential { return "差异" } @@ -591,15 +602,22 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No if target == nil { return nil, apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil) } - configRaw, err := s.cipher.Decrypt(target.ConfigCiphertext) - if err != nil { - return nil, fmt.Errorf("decrypt storage config: %w", err) - } // 拆开 sourcePaths sourcePaths := []string{} if strings.TrimSpace(task.SourcePaths) != "" { _ = json.Unmarshal([]byte(task.SourcePaths), &sourcePaths) } + transferMode := storage.TransferModeDirect + if backupRecord.StorageTransferMode == storage.TransferModeMasterRelay { + transferMode = storage.TransferModeMasterRelay + } + var configRaw []byte + if transferMode == storage.TransferModeDirect { + configRaw, err = s.cipher.Decrypt(target.ConfigCiphertext) + if err != nil { + return nil, fmt.Errorf("decrypt storage config: %w", err) + } + } return &AgentRestoreSpec{ RestoreRecordID: restore.ID, BackupRecordID: backupRecord.ID, @@ -618,10 +636,11 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No Compression: task.Compression, Encrypt: task.Encrypt, Storage: AgentStorageTargetConfig{ - ID: target.ID, - Type: target.Type, - Name: target.Name, - Config: json.RawMessage(configRaw), + ID: target.ID, + Type: target.Type, + Name: target.Name, + Config: json.RawMessage(configRaw), + TransferMode: transferMode, }, StoragePath: backupRecord.StoragePath, FileName: backupRecord.FileName, @@ -629,6 +648,63 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No }, nil } +type AgentArtifactDownload struct { + Reader io.ReadCloser + Size int64 +} + +// DownloadAgentArtifact opens a Master-local object for authenticated streaming +// back to the Agent that owns the restore record. +func (s *RestoreService) DownloadAgentArtifact(ctx context.Context, node *model.Node, restoreID uint) (*AgentArtifactDownload, error) { + if node == nil { + return nil, apperror.Unauthorized("RESTORE_RECORD_FORBIDDEN", "恢复记录不属于当前节点", nil) + } + restore, err := s.restores.FindByID(ctx, restoreID) + if err != nil { + return nil, err + } + if restore == nil { + return nil, apperror.New(404, "RESTORE_RECORD_NOT_FOUND", "恢复记录不存在", nil) + } + if restore.NodeID != node.ID { + return nil, apperror.Unauthorized("RESTORE_RECORD_FORBIDDEN", "恢复记录不属于当前节点", nil) + } + if isRestoreRecordTerminal(restore.Status) { + return nil, apperror.BadRequest("RESTORE_RECORD_TERMINAL", "恢复记录已结束，不能继续下载产物", nil) + } + record, err := s.records.FindByID(ctx, restore.BackupRecordID) + if err != nil { + return nil, err + } + if record == nil { + return nil, apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "源备份记录不存在", nil) + } + target, err := s.targets.FindByID(ctx, record.StorageTargetID) + if err != nil { + return nil, err + } + if target == nil || !strings.EqualFold(target.Type, storage.TypeLocalDisk) || record.StorageTransferMode != storage.TransferModeMasterRelay { + return nil, apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "该存储目标应由 Agent 直接下载", nil) + } + configMap := map[string]any{} + if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil { + return nil, fmt.Errorf("decrypt storage config: %w", err) + } + provider, err := s.storageRegistry.Create(ctx, target.Type, configMap) + if err != nil { + return nil, fmt.Errorf("create master relay provider: %w", err) + } + reader, err := provider.Download(ctx, record.StoragePath) + if err != nil { + return nil, fmt.Errorf("open master relay artifact: %w", err) + } + size := record.FileSize + if size <= 0 { + size = -1 + } + return &AgentArtifactDownload{Reader: reader, Size: size}, nil +} + // UpdateAgentRestore Agent 回传状态/日志。 func (s *RestoreService) UpdateAgentRestore(ctx context.Context, node *model.Node, restoreID uint, update AgentRestoreUpdate) error { restore, err := s.restores.FindByID(ctx, restoreID) diff --git a/server/internal/service/restore_service_test.go b/server/internal/service/restore_service_test.go index 5991c9c..a411728 100644 --- a/server/internal/service/restore_service_test.go +++ b/server/internal/service/restore_service_test.go @@ -1,8 +1,10 @@ package service import ( + "bytes" "context" "encoding/json" + "io" "os" "path/filepath" "strings" @@ -427,16 +429,27 @@ func TestRestoreServiceAgentRestoreAccessUsesRestoreRecordNode(t *testing.T) { } startedAt := time.Now().UTC() completedAt := startedAt.Add(time.Second) + artifact := []byte("central backup artifact") + storagePath := "file/2026/05/09/remote.tar.gz" + artifactPath := filepath.Join(h.storageDir, filepath.FromSlash(storagePath)) + if err := os.MkdirAll(filepath.Dir(artifactPath), 0o755); err != nil { + t.Fatalf("MkdirAll artifact parent: %v", err) + } + if err := os.WriteFile(artifactPath, artifact, 0o600); err != nil { + t.Fatalf("WriteFile artifact: %v", err) + } backupRecord := &model.BackupRecord{ - TaskID: task.ID, - StorageTargetID: task.StorageTargetID, - NodeID: owner.ID, - Status: model.BackupRecordStatusSuccess, - FileName: "remote.tar.gz", - StoragePath: "file/2026/05/09/remote.tar.gz", - Checksum: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - StartedAt: startedAt, - CompletedAt: &completedAt, + TaskID: task.ID, + StorageTargetID: task.StorageTargetID, + NodeID: owner.ID, + Status: model.BackupRecordStatusSuccess, + FileName: "remote.tar.gz", + StoragePath: storagePath, + FileSize: int64(len(artifact)), + StorageTransferMode: storage.TransferModeMasterRelay, + Checksum: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + StartedAt: startedAt, + CompletedAt: &completedAt, } if err := h.records.Create(ctx, backupRecord); err != nil { t.Fatalf("Create backup record: %v", err) @@ -464,6 +477,21 @@ func TestRestoreServiceAgentRestoreAccessUsesRestoreRecordNode(t *testing.T) { if spec.Checksum != backupRecord.Checksum { t.Fatalf("expected spec.Checksum=%q, got %q", backupRecord.Checksum, spec.Checksum) } + if spec.Storage.TransferMode != storage.TransferModeMasterRelay { + t.Fatalf("expected Master relay restore, got %#v", spec.Storage) + } + download, err := h.service.DownloadAgentArtifact(ctx, owner, restore.ID) + if err != nil { + t.Fatalf("DownloadAgentArtifact returned error: %v", err) + } + downloaded, readErr := io.ReadAll(download.Reader) + closeErr := download.Reader.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("read relayed restore artifact: read=%v close=%v", readErr, closeErr) + } + if !bytes.Equal(downloaded, artifact) { + t.Fatalf("relayed restore artifact differs: %q", downloaded) + } if _, err := h.service.GetAgentRestoreSpec(ctx, other, restore.ID); err == nil { t.Fatal("expected non-owner node to be forbidden from restore spec") } diff --git a/server/internal/service/verification_service.go b/server/internal/service/verification_service.go index 4350d8c..45e7f40 100644 --- a/server/internal/service/verification_service.go +++ b/server/internal/service/verification_service.go @@ -299,6 +299,20 @@ func (s *VerificationService) executeLocally(ctx context.Context, verID uint, ta logger.Errorf("创建存储客户端失败：%v", err) return } + if backupRecord.BackupKind == model.BackupKindRepository { + logger.Infof("验证 CDC 仓库快照及全部引用块：%s", backupRecord.StoragePath) + report, verifyErr := backup.NewRepositoryStore(s.cipher.Key()).Verify(ctx, provider, backupRecord.StoragePath, backupRecord.Checksum) + if verifyErr != nil { + errMessage = verifyErr.Error() + summary = "CDC 仓库完整性校验失败" + logger.Errorf("验证未通过：%v", verifyErr) + return + } + status = model.VerificationRecordStatusSuccess + summary = fmt.Sprintf("CDC 仓库完整性校验通过：%d 个条目、%d 个唯一块、%d bytes", report.Entries, report.Chunks, report.Bytes) + logger.Infof("%s", summary) + return + } fileName := backupRecord.FileName if strings.TrimSpace(fileName) == "" { fileName = filepath.Base(backupRecord.StoragePath) diff --git a/server/internal/storage/rclone/provider.go b/server/internal/storage/rclone/provider.go index 60f1257..ff33147 100644 --- a/server/internal/storage/rclone/provider.go +++ b/server/internal/storage/rclone/provider.go @@ -2,6 +2,7 @@ package rclone import ( "context" + "errors" "fmt" "io" "sort" @@ -68,13 +69,56 @@ func (p *Provider) Download(ctx context.Context, objectKey string) (io.ReadClose return reader, nil } +// DownloadRange reads one slice from an object. Most object-storage backends +// map this to a native HTTP Range request. Backends that reject ranged reads +// fall back to a full stream while preserving the same interface contract. +func (p *Provider) DownloadRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("rclone download range %s: invalid offset=%d length=%d", objectKey, offset, length) + } + obj, err := p.rfs.NewObject(ctx, objectKey) + if err != nil { + return nil, fmt.Errorf("rclone find object %s: %w", objectKey, err) + } + reader, rangeErr := obj.Open(ctx, &fs.RangeOption{Start: offset, End: offset + length - 1}) + if rangeErr == nil { + return reader, nil + } + reader, err = obj.Open(ctx) + if err != nil { + return nil, fmt.Errorf("rclone download range %s (range: %v; fallback: %w)", objectKey, rangeErr, err) + } + if offset > 0 { + if _, err := io.CopyN(io.Discard, reader, offset); err != nil { + closeErr := reader.Close() + return nil, errors.Join(fmt.Errorf("rclone seek object %s: %w", objectKey, err), closeErr) + } + } + return &limitedReadCloser{Reader: io.LimitReader(reader, length), closer: reader}, nil +} + +type limitedReadCloser struct { + io.Reader + closer io.Closer +} + +func (r *limitedReadCloser) Close() error { + return r.closer.Close() +} + // Delete 通过 rclone 删除远端对象。 func (p *Provider) Delete(ctx context.Context, objectKey string) error { obj, err := p.rfs.NewObject(ctx, objectKey) if err != nil { + if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorDirNotFound) { + return nil + } return fmt.Errorf("rclone find object %s: %w", objectKey, err) } if err := obj.Remove(ctx); err != nil { + if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorDirNotFound) { + return nil + } return fmt.Errorf("rclone delete %s: %w", objectKey, err) } return nil @@ -102,6 +146,9 @@ func (p *Provider) List(ctx context.Context, prefix string) ([]storage.ObjectInf return nil }) if err != nil { + if errors.Is(err, fs.ErrorDirNotFound) || errors.Is(err, fs.ErrorObjectNotFound) { + return []storage.ObjectInfo{}, nil + } return nil, fmt.Errorf("rclone list %s: %w", prefix, err) } return items, nil diff --git a/server/internal/storage/types.go b/server/internal/storage/types.go index 92672e7..aae226a 100644 --- a/server/internal/storage/types.go +++ b/server/internal/storage/types.go @@ -34,6 +34,14 @@ const ( TypeFTP = string(ProviderTypeFTP) ) +const ( + // TransferModeDirect lets an Agent write to a network-accessible backend. + TransferModeDirect = "direct" + // TransferModeMasterRelay streams an artifact through the authenticated + // Agent API so a remote source can use storage mounted only on the Master. + TransferModeMasterRelay = "master_relay" +) + type ObjectInfo struct { Key string `json:"key"` Size int64 `json:"size"` @@ -49,6 +57,13 @@ type StorageProvider interface { List(ctx context.Context, prefix string) ([]ObjectInfo, error) } +// StorageRangeDownloader is an optional capability used by packed repository +// backups. Implementations return exactly the requested byte range when the +// backend supports ranged reads and may transparently fall back to a full read. +type StorageRangeDownloader interface { + DownloadRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error) +} + type ProviderFactory interface { Type() ProviderType } @@ -92,7 +107,8 @@ func ParseProviderType(value string) ProviderType { } type LocalDiskConfig struct { - BasePath string `json:"basePath"` + BasePath string `json:"basePath"` + MasterRelay bool `json:"masterRelay"` } type S3Config struct { @@ -151,4 +167,3 @@ type FTPConfig struct { type StorageDirCleaner interface { RemoveEmptyDirs(ctx context.Context, prefix string) error } - diff --git a/web/src/components/backup-records/BackupRecordLogDrawer.tsx b/web/src/components/backup-records/BackupRecordLogDrawer.tsx index b0e35c6..9380ef6 100644 --- a/web/src/components/backup-records/BackupRecordLogDrawer.tsx +++ b/web/src/components/backup-records/BackupRecordLogDrawer.tsx @@ -280,6 +280,10 @@ export function BackupRecordLogDrawer({ visible, recordId, onCancel, onChanged } { label: '文件名', value: record.fileName || '-' }, { label: '文件大小', value: formatBytes(record.fileSize) }, { label: '存储路径', value: record.storagePath || '-' }, + ...(record.storageTransferMode ? [{ + label: '传输路径', + value: record.storageTransferMode === 'master_relay' ? 'Master 流式中转' : 'Agent 直传', + }] : []), { label: '开始时间', value: formatDateTime(record.startedAt) }, { label: '完成时间', value: formatDateTime(record.completedAt) }, { label: '耗时', value: formatDuration(record.durationSeconds) }, @@ -316,14 +320,16 @@ export function BackupRecordLogDrawer({ visible, recordId, onCancel, onChanged } )} - {record.storageUploadResults && record.storageUploadResults.length > 1 && ( + {record.storageUploadResults && (record.storageUploadResults.length > 1 || record.storageUploadResults.some((result) => result.transferMode)) && (
存储目标上传结果 ({ label: r.storageTargetName, - value: r.status === 'success' ? '上传成功' : `上传失败: ${r.error || '未知错误'}`, + value: r.status === 'success' + ? `上传成功${r.transferMode === 'master_relay' ? ' · Master 流式中转' : r.transferMode === 'direct' ? ' · Agent 直传' : ''}` + : `上传失败: ${r.error || '未知错误'}`, }))} />
diff --git a/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx b/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx index 7a24079..71ee7f4 100644 --- a/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx +++ b/web/src/components/backup-tasks/BackupTaskFormDrawer.tsx @@ -1,5 +1,5 @@ import { Alert, Button, Divider, Drawer, Input, InputNumber, Select, Space, Steps, Switch, Typography, Grid } from '@arco-design/web-react' -import { IconDelete, IconPlus } from '@arco-design/web-react/icon' +import { IconDelete, IconPlus } from '../icons' import { useEffect, useMemo, useState } from 'react' import { CronInput } from '../CronInput' import type { StorageTargetDetail, StorageTargetPayload, StorageTargetSummary } from '../../types/storage-targets' @@ -9,6 +9,8 @@ import type { NodeSummary } from '../../types/nodes' import { DatabasePicker } from '../common/DatabasePicker' import { DirectoryPicker } from '../common/DirectoryPicker' import { StorageTargetFormDrawer } from '../storage-targets/StorageTargetFormDrawer' +import { StorageTargetName } from '../storage-targets/StorageTargetName' +import { SourceServerSelector } from './SourceServerSelector' import { backupCompressionOptions, backupTaskTypeOptions, @@ -168,7 +170,7 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa return 0 }) return sorted.map((item) => ({ - label: item.starred ? `★ ${item.name}` : item.name, + label: , value: item.id, disabled: !item.enabled, })) @@ -176,21 +178,6 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa [storageTargets], ) - // 执行节点选项：本地节点显示 "本机 (local)"，远程节点带状态后缀 - const nodeOptions = useMemo(() => { - const list = nodes ?? [] - return [ - { label: '本机 (Master)', value: 0 }, - ...list - .filter((item) => !item.isLocal) - .map((item) => ({ - label: `${item.name}${item.status === 'online' ? '' : '（离线）'}`, - value: item.id, - disabled: item.status !== 'online', - })), - ] - }, [nodes]) - function updateDraft(patch: Partial) { setDraft((current) => ({ ...current, ...patch })) } @@ -251,6 +238,12 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa if (validPaths.length === 0 && !value.sourcePath.trim()) { return '请输入至少一个源路径' } + if (value.backupMode === 'repository' && (((value.nodeId ?? 0) > 0 && value.nodeId !== localNodeId) || value.nodePoolTag?.trim())) { + return 'CDC 仓库模式当前仅支持 Master 本机执行' + } + if (value.backupMode === 'repository' && value.replicationTargetIds.length > 0) { + return 'CDC 仓库模式请直接多选存储目标，不能使用对象级副本复制' + } } if (isSQLiteBackupTask(value.type) && !value.dbPath.trim()) { return '请输入 SQLite 数据库路径' @@ -306,33 +299,26 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa 备份类型 { - const nodeId = Number(value ?? 0) - // 固定节点与节点池互斥：切到固定节点时清空 NodePoolTag - updateDraft(nodeId > 0 ? { nodeId, nodePoolTag: '' } : { nodeId }) - }} - /> - - 任务在所选节点上执行备份与恢复；源路径/数据库以该节点视角解析。远程节点需先在"节点管理"中安装 Agent。 - - -
- 节点池标签（可选） - 0} - onChange={(value) => updateDraft({ nodePoolTag: value })} - /> - - 执行节点选"本机 / 未指定"时可启用；从节点 Labels 命中此 tag 的在线节点中按当前运行任务数最少的挑选一台执行。 - -
+ { + // 固定源服务器与服务器池互斥；CDC 仓库仍固定在 Master 单写者。 + updateDraft(nodeId > 0 + ? { + nodeId, + nodePoolTag: '', + backupMode: nodeId !== localNodeId && draft.backupMode === 'repository' ? 'full' : draft.backupMode, + } + : { nodeId }) + }} + onNodePoolTagChange={(value) => updateDraft({ + nodePoolTag: value, + backupMode: value.trim() && draft.backupMode === 'repository' ? 'full' : draft.backupMode, + })} + />
Cron 表达式 updateDraft({ cronExpr: value })} /> @@ -587,6 +573,11 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa )} + {((draft.nodeId ?? 0) > 0 && draft.nodeId !== localNodeId) || draft.nodePoolTag?.trim() ? ( + + 远程源服务器会直传 S3、WebDAV 等网络存储；本地磁盘目标启用 Master 中转后，文件经 Agent 认证 API 流式写入中央目录。跨公网部署请为 Master 配置 HTTPS。 + + ) : null}
压缩策略 @@ -600,8 +591,11 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa options={[ { label: '全量备份', value: 'full' }, { label: '差异备份（仅文件、本机）', value: 'differential' }, + { label: 'CDC 去重仓库（仅文件、本机）', value: 'repository' }, ]} - onChange={(value) => updateDraft({ backupMode: value as BackupMode })} + onChange={(value) => updateDraft(value === 'repository' + ? { backupMode: value as BackupMode, nodeId: 0, nodePoolTag: '', replicationTargetIds: [] } + : { backupMode: value as BackupMode })} /> {draft.backupMode === 'differential' && (
@@ -618,6 +612,11 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa />
)} + {draft.backupMode === 'repository' && ( + + 文件按内容边界切块并写入全局分块池；相同数据跨文件、跨快照只上传一次。新块会合并为 pack，恢复时通过索引按需读取。当前版本采用单写者索引，因此固定在 Master 本机执行；需要多副本时请直接多选上方存储目标。 + + )}
)}
@@ -736,10 +735,13 @@ export function BackupTaskFormDrawer({ visible, loading, initialValue, storageTa value={draft.replicationTargetIds} placeholder="选择副本目标（不选 = 不启用复制）" options={storageTargetOptions.filter((opt) => !(draft.storageTargetIds ?? []).includes(opt.value as number))} + disabled={draft.backupMode === 'repository'} onChange={(values: number[]) => updateDraft({ replicationTargetIds: values })} /> - 备份成功后自动镜像到副本存储。满足 3-2-1 规则：至少 2 份副本、至少 1 份异地。建议选不同 provider 的目标。 + {draft.backupMode === 'repository' + ? 'CDC 仓库包含共享 pack 与索引，不能只复制单个快照对象；请在“存储目标”中直接多选以生成完整仓库副本。' + : '备份成功后自动镜像到副本存储。满足 3-2-1 规则：至少 2 份副本、至少 1 份异地。建议选不同 provider 的目标。'}
diff --git a/web/src/components/backup-tasks/SourceServerSelector.test.ts b/web/src/components/backup-tasks/SourceServerSelector.test.ts new file mode 100644 index 0000000..23baa10 --- /dev/null +++ b/web/src/components/backup-tasks/SourceServerSelector.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' +import type { NodeSummary } from '../../types/nodes' +import { buildSourceServerOptions } from './SourceServerSelector' + +function node(id: number, name: string, status: NodeSummary['status'], isLocal = false): NodeSummary { + return { + id, + name, + status, + isLocal, + hostname: '', + ipAddress: '', + os: '', + arch: '', + agentVersion: '', + lastSeen: '', + createdAt: '', + } +} + +describe('buildSourceServerOptions', () => { + it('keeps Master first and disables offline remote sources', () => { + const options = buildSourceServerOptions([ + node(1, 'local', 'online', true), + node(2, 'source-b', 'online'), + node(3, 'source-c', 'offline'), + ]) + + expect(options).toEqual([ + { label: 'Master 本机', value: 0, disabled: false }, + { label: 'source-b', value: 2, disabled: false }, + { label: 'source-c（离线）', value: 3, disabled: true }, + ]) + }) +}) diff --git a/web/src/components/backup-tasks/SourceServerSelector.tsx b/web/src/components/backup-tasks/SourceServerSelector.tsx new file mode 100644 index 0000000..86b5396 --- /dev/null +++ b/web/src/components/backup-tasks/SourceServerSelector.tsx @@ -0,0 +1,57 @@ +import { Input, Select, Typography } from '@arco-design/web-react' +import { useMemo } from 'react' +import type { NodeSummary } from '../../types/nodes' + +interface SourceServerSelectorProps { + nodeId: number + nodePoolTag: string + localNodeId?: number + nodes?: NodeSummary[] + onNodeChange: (nodeId: number) => void + onNodePoolTagChange: (tag: string) => void +} + +export function buildSourceServerOptions(nodes: NodeSummary[] = []) { + return [ + { label: 'Master 本机', value: 0, disabled: false }, + ...nodes + .filter((node) => !node.isLocal) + .map((node) => ({ + label: `${node.name}${node.status === 'online' ? '' : '（离线）'}`, + value: node.id, + disabled: node.status !== 'online', + })), + ] +} + +export function SourceServerSelector({ nodeId, nodePoolTag, localNodeId, nodes, onNodeChange, onNodePoolTagChange }: SourceServerSelectorProps) { + const options = useMemo(() => buildSourceServerOptions(nodes), [nodes]) + const selectedNode = nodes?.find((node) => node.id === nodeId) + const isRemote = nodeId > 0 && nodeId !== localNodeId + + return ( + <> +
+ 源服务器 + 0} + onChange={onNodePoolTagChange} + /> + + 仅在选择 Master 本机时可填写；系统从 Labels 命中该标签的在线 Agent 中选择当前运行任务最少的一台。 + +
+ + ) +} diff --git a/web/src/components/common/DirectoryPicker.tsx b/web/src/components/common/DirectoryPicker.tsx index c24b15a..5129868 100644 --- a/web/src/components/common/DirectoryPicker.tsx +++ b/web/src/components/common/DirectoryPicker.tsx @@ -1,5 +1,5 @@ import { Button, Input, Message, Modal, Space, Spin, Tree, Typography, Empty } from '@arco-design/web-react' -import { IconFolder, IconFile, IconFolderAdd } from '@arco-design/web-react/icon' +import { IconFolder, IconFile, IconFolderAdd } from '../icons' import { useCallback, useState } from 'react' import { listNodeDirectory } from '../../services/nodes' import type { DirEntry } from '../../types/nodes' diff --git a/web/src/components/common/EventCenter.tsx b/web/src/components/common/EventCenter.tsx index 42561cf..fa20f93 100644 --- a/web/src/components/common/EventCenter.tsx +++ b/web/src/components/common/EventCenter.tsx @@ -1,5 +1,5 @@ import { Badge, Button, Drawer, Empty, Notification, Space, Tag, Typography } from '@arco-design/web-react' -import { IconNotification } from '@arco-design/web-react/icon' +import { IconNotification } from '../icons' import { useEffect, useState } from 'react' import { useEventStream, type SystemEvent } from '../../hooks/useEventStream' import { useEventStore } from '../../stores/events' diff --git a/web/src/components/common/GlobalSearch.tsx b/web/src/components/common/GlobalSearch.tsx index 3a319a2..dcfc2bf 100644 --- a/web/src/components/common/GlobalSearch.tsx +++ b/web/src/components/common/GlobalSearch.tsx @@ -1,5 +1,5 @@ import { Empty, Input, Modal, Space, Spin, Tag, Typography } from '@arco-design/web-react' -import { IconSearch } from '@arco-design/web-react/icon' +import { IconSearch } from '../icons' import { useCallback, useEffect, useRef, useState } from 'react' import { useNavigate } from 'react-router-dom' import { globalSearch, type SearchKind, type SearchResult, type SearchResultItem } from '../../services/search' diff --git a/web/src/components/common/LanguageSwitcher.tsx b/web/src/components/common/LanguageSwitcher.tsx new file mode 100644 index 0000000..10a89f8 --- /dev/null +++ b/web/src/components/common/LanguageSwitcher.tsx @@ -0,0 +1,18 @@ +import { Select } from '@arco-design/web-react' +import { useTranslation } from 'react-i18next' +import { languageOptions, normalizeLanguage, setApplicationLanguage, type SupportedLanguage } from '../../i18n' + +export function LanguageSwitcher() { + const { t, i18n } = useTranslation() + const currentLanguage = normalizeLanguage(i18n.resolvedLanguage) + + return ( + } size="large" /> + + } size="large" /> - - } size="large" /> + + } size="large" /> - - } size="large" /> + + } size="large" /> - ) : ( form={loginForm} layout="vertical" onSubmit={handleLogin}> - - } size="large" onChange={resetTwoFactorPrompt} /> + + } size="large" onChange={resetTwoFactorPrompt} /> - - } size="large" onChange={resetTwoFactorPrompt} /> + + } size="large" onChange={resetTwoFactorPrompt} /> {twoFactorRequired && ( <> - - } size="large" maxLength={32} /> + + } size="large" maxLength={32} /> - - - + + + - 信任此设备 30 天 + {t('auth.trustDevice')} )} - )} diff --git a/web/src/pages/nodes/BatchCommandTable.tsx b/web/src/pages/nodes/BatchCommandTable.tsx index b96f82d..c724e45 100644 --- a/web/src/pages/nodes/BatchCommandTable.tsx +++ b/web/src/pages/nodes/BatchCommandTable.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react' import { Table, Button, Space, Message, Typography, Tag } from '@arco-design/web-react' -import { IconCopy, IconDownload, IconRefresh } from '@arco-design/web-react/icon' +import { IconCopy, IconDownload, IconRefresh } from '../../components/icons' const { Text } = Typography diff --git a/web/src/pages/nodes/NodesPage.tsx b/web/src/pages/nodes/NodesPage.tsx index 9587078..bdcd079 100644 --- a/web/src/pages/nodes/NodesPage.tsx +++ b/web/src/pages/nodes/NodesPage.tsx @@ -5,7 +5,7 @@ import { } from '@arco-design/web-react' import { IconPlus, IconDelete, IconDesktop, IconCloudDownload, IconEdit, IconMore, -} from '@arco-design/web-react/icon' +} from '../../components/icons' import type { NodeSummary } from '../../types/nodes' import { listNodes, deleteNode, updateNode, rotateNodeToken } from '../../services/nodes' import { fetchSystemInfo } from '../../services/system' diff --git a/web/src/pages/nodes/wizard/Step3CommandPreview.tsx b/web/src/pages/nodes/wizard/Step3CommandPreview.tsx index 9000060..455119f 100644 --- a/web/src/pages/nodes/wizard/Step3CommandPreview.tsx +++ b/web/src/pages/nodes/wizard/Step3CommandPreview.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react' import { Typography, Button, Space, Collapse, Spin, Message, Tag } from '@arco-design/web-react' -import { IconCopy, IconRefresh } from '@arco-design/web-react/icon' +import { IconCopy, IconRefresh } from '../../../components/icons' import { fetchScriptPreview } from '../../../services/nodes' import type { InstallTokenResult, InstallMode } from '../../../types/nodes' import { buildAgentDownloadCommand, buildAgentInstallCommand, buildEmbeddedAgentInstallCommand } from '../installCommands' diff --git a/web/src/pages/reports/ReportsPage.tsx b/web/src/pages/reports/ReportsPage.tsx index d26d6d7..d6fec7b 100644 --- a/web/src/pages/reports/ReportsPage.tsx +++ b/web/src/pages/reports/ReportsPage.tsx @@ -1,5 +1,5 @@ import { Button, Card, Grid, Message, Select, Space, Statistic, Table, Tag, Typography } from '@arco-design/web-react' -import { IconDownload, IconRefresh } from '@arco-design/web-react/icon' +import { IconDownload, IconRefresh } from '../../components/icons' import { useCallback, useEffect, useState } from 'react' import { downloadComplianceCSV, fetchComplianceReport } from '../../services/reports' import type { ComplianceReport, ComplianceRisk, ComplianceTaskRow } from '../../types/reports' diff --git a/web/src/pages/storage-targets/StorageTargetsPage.tsx b/web/src/pages/storage-targets/StorageTargetsPage.tsx index c987f0a..7e062d1 100644 --- a/web/src/pages/storage-targets/StorageTargetsPage.tsx +++ b/web/src/pages/storage-targets/StorageTargetsPage.tsx @@ -18,6 +18,7 @@ import { formatBytes } from '../../utils/format' import type { StorageConnectionTestResult, StorageTargetDetail, StorageTargetPayload, StorageTargetSummary } from '../../types/storage-targets' import { getStorageTargetTypeLabel } from '../../components/storage-targets/field-config' import { StorageTargetFormDrawer } from '../../components/storage-targets/StorageTargetFormDrawer' +import { StorageTargetName } from '../../components/storage-targets/StorageTargetName' function resolveErrorMessage(error: unknown) { if (axios.isAxiosError(error)) { @@ -224,7 +225,7 @@ export function StorageTargetsPage() {
- {target.starred ? '★ ' : ''}{target.name} + {getStorageTargetTypeLabel(target.type) && {getStorageTargetTypeLabel(target.type)}} diff --git a/web/src/styles/global.css b/web/src/styles/global.css index 3b58508..cf28e30 100644 --- a/web/src/styles/global.css +++ b/web/src/styles/global.css @@ -42,48 +42,25 @@ body { .login-bg { position: fixed; inset: 0; - background: linear-gradient(135deg, #111a2c 0%, #1f2d47 100%); + background: #111a2c; z-index: 0; } -.login-bg::before { - content: ''; - position: absolute; - width: 800px; - height: 800px; - border-radius: 50%; - background: radial-gradient(circle, rgba(52,145,250,0.08) 0%, transparent 70%); - top: -300px; - right: -200px; -} - -.login-bg::after { - content: ''; - position: absolute; - width: 600px; - height: 600px; - border-radius: 50%; - background: radial-gradient(circle, rgba(114,46,209,0.06) 0%, transparent 70%); - bottom: -200px; - left: -100px; -} - .login-container { display: flex; width: 1000px; max-width: 90vw; min-height: 560px; background: var(--color-bg-2); - border-radius: 20px; + border-radius: 4px; overflow: hidden; - box-shadow: 0 20px 60px rgba(0,0,0,0.4); z-index: 1; animation: slideUp 0.6s cubic-bezier(0.34, 1.56, 0.64, 1); } .login-banner { flex: 1; - background: linear-gradient(135deg, var(--color-primary-6, #165dff) 0%, var(--color-primary-8, #0e42d2) 100%); + background: var(--color-primary-6, #165dff); position: relative; display: flex; align-items: center; diff --git a/web/src/test/setup.ts b/web/src/test/setup.ts index a866f69..a9a4ac5 100644 --- a/web/src/test/setup.ts +++ b/web/src/test/setup.ts @@ -24,3 +24,17 @@ Object.defineProperty(window, 'localStorage', { value: storage, configurable: true, }) + +Object.defineProperty(window, 'matchMedia', { + configurable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addEventListener: () => undefined, + removeEventListener: () => undefined, + addListener: () => undefined, + removeListener: () => undefined, + dispatchEvent: () => false, + }), +}) diff --git a/web/src/types/backup-records.ts b/web/src/types/backup-records.ts index 01c980f..a6debfc 100644 --- a/web/src/types/backup-records.ts +++ b/web/src/types/backup-records.ts @@ -21,12 +21,13 @@ export interface BackupRecordSummary { fileSize: number checksum: string storagePath: string + storageTransferMode?: 'direct' | 'master_relay' durationSeconds: number errorMessage: string startedAt: string completedAt?: string locked: boolean - backupKind: 'full' | 'differential' + backupKind: 'full' | 'differential' | 'repository' } export interface BackupRecordContentEntry { @@ -49,6 +50,7 @@ export interface StorageUploadResultItem { status: 'success' | 'failed' storagePath?: string fileSize?: number + transferMode?: 'direct' | 'master_relay' error?: string } diff --git a/web/src/types/backup-tasks.ts b/web/src/types/backup-tasks.ts index b254ad1..eeeeb20 100644 --- a/web/src/types/backup-tasks.ts +++ b/web/src/types/backup-tasks.ts @@ -1,7 +1,7 @@ export type BackupTaskType = 'file' | 'mysql' | 'sqlite' | 'postgresql' | 'saphana' | 'mongodb' export type BackupTaskStatus = 'idle' | 'running' | 'success' | 'failed' export type BackupCompression = 'gzip' | 'zstd' | 'none' -export type BackupMode = 'full' | 'differential' +export type BackupMode = 'full' | 'differential' | 'repository' export interface BackupTaskSummary { id: number