diff --git a/.github/assets/logo-dark.svg b/.github/assets/logo-dark.svg new file mode 100644 index 000000000..be60e71de --- /dev/null +++ b/.github/assets/logo-dark.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/.github/assets/logo-light.svg b/.github/assets/logo-light.svg new file mode 100644 index 000000000..e498602ca --- /dev/null +++ b/.github/assets/logo-light.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/.github/assets/v2-video.jpg b/.github/assets/v2-video.jpg new file mode 100644 index 000000000..2df1537b0 Binary files /dev/null and b/.github/assets/v2-video.jpg differ diff --git a/.github/workflows/binary.yaml b/.github/workflows/binary.yaml index 0ceb75011..849225820 100644 --- a/.github/workflows/binary.yaml +++ b/.github/workflows/binary.yaml @@ -53,9 +53,9 @@ jobs: ref: ${{ inputs.ref }} - name: Setup Go - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: 1.24.11 + go-version: 1.27.2 - name: Resolve the version id: resolve @@ -76,6 +76,8 @@ jobs: VERSION: ${{ steps.resolve.outputs.version }} run: | set -euo pipefail + # dist/ is gitignored, so it does not exist in a fresh checkout. + mkdir -p dist for arch in amd64 arm64; do GOARCH="$arch" go build \ -o "dist/wings_linux_${arch}" \ diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 9fb6a6b42..e61491fca 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -29,10 +29,10 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@0499de31b99561a6d14a36a5f662c2a54f91beee # v3.29.5 + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@0499de31b99561a6d14a36a5f662c2a54f91beee # v3.29.5 + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 58438626a..3143c563c 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -92,7 +92,7 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Login to GitHub Container Registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -100,7 +100,7 @@ jobs: - name: Build and push id: build - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . file: ./Dockerfile diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..ac50ffa96 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,29 @@ +name: Lint + +on: + push: + branches: + - develop + pull_request: + branches: + - develop + +jobs: + golangci-lint: + name: golangci-lint + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Code checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.26.9" + + - name: golangci-lint + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.11.4 diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index c6b088fa3..d94e61e6a 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04] - go: ["1.24.11", "1.25.5"] + go: ["1.26.9", "1.27.2"] goos: [linux] goarch: [amd64, arm64] permissions: @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Setup Go - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ matrix.go }} @@ -62,15 +62,15 @@ jobs: go test -race $(go list ./...) - name: Upload Release Artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.24.11' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.27.2' }} with: name: wings_linux_${{ matrix.goarch }} path: dist/wings - name: Upload Debug Artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.24.11' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.27.2' }} with: name: wings_linux_${{ matrix.goarch }}_debug path: dist/wings_debug diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8537430c1..128756cdc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -240,7 +240,7 @@ jobs: packages: write steps: - name: Login to GitHub Container Registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.gitignore b/.gitignore index c636920e7..76f2be500 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,7 @@ pprof.* .claude-flow/ .hive-mind/ CLAUDE.md + +# Local build artifacts (fork): never commit binaries +/wings-* +/dist/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..b92e86b8b --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,36 @@ +version: "2" + +# Wings only builds for Linux; run with GOOS=linux (see `make lint`) so the +# _linux/_unix files are type-checked and linted. + +linters: + # errcheck, govet, ineffassign, staticcheck, unused + default: standard + enable: + - gosec + settings: + gosec: + excludes: + - G104 # unhandled errors, already reported by errcheck + - G115 # integer overflow on conversion, too noisy for stat/syscall code + exclusions: + generated: lax + presets: + - std-error-handling + rules: + - path: _test\.go + linters: + - gosec + # Explicit embedded-field selectors (e.g. f.File.Seek in a wrapper type) + # are kept on purpose. + - linters: + - staticcheck + text: "QF1008:" + +formatters: + enable: + - gofmt + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/BACKUP_PROGRESS_EVENTS.md b/BACKUP_PROGRESS_EVENTS.md index 00ca34cbf..87784291a 100644 --- a/BACKUP_PROGRESS_EVENTS.md +++ b/BACKUP_PROGRESS_EVENTS.md @@ -7,13 +7,14 @@ This document describes the new real-time backup progress events available via W ### Event Name - **Event**: `backup progress` - **Type**: Real-time progress updates -- **Frequency**: ~2 updates per second maximum (intelligent throttling) +- **Frequency**: up to 4 updates per second (250 ms throttle) ## Event Structure ```json { - "type": "create|restore", + "backup_id": "47363ce7-d70a-430e-8e75-6dc87c8d016d", + "type": "create|restore|download", "percentage": 0-100, "bytes_written": 1234567, "bytes_total": 10000000 @@ -24,10 +25,11 @@ This document describes the new real-time backup progress events available via W | Field | Type | Description | |-------|------|-------------| -| `type` | string | Type of operation: `"create"` for backup creation, `"restore"` for backup restoration | +| `backup_id` | string | UUID of the backup the update belongs to | +| `type` | string | Type of operation: `"create"` for backup creation, `"restore"` for backup restoration, `"download"` while an S3 backup is downloaded for a restore (sent unthrottled from the download reader) | | `percentage` | int | Progress percentage (0-100). For restore operations, this is typically `0` | -| `bytes_written` | int64 | Number of bytes processed so far | -| `bytes_total` | int64 | Total bytes to process (for create operations) | +| `bytes_written` | int64 | Number of bytes processed so far (omitted when 0) | +| `bytes_total` | int64 | Total bytes to process (omitted when unknown or 0) | **Special Values:** - `percentage: 100` = Operation completed successfully @@ -200,7 +202,7 @@ function BackupProgressComponent({ websocket }) { ## Performance Characteristics -- **Update Frequency**: Maximum ~2 updates per second per operation +- **Update Frequency**: Maximum 4 updates per second per operation (250 ms throttle) - **Bandwidth Usage**: ~100 bytes per update - **CPU Overhead**: <0.00001% of backup operation time - **Memory Usage**: ~80 bytes per active backup operation @@ -210,7 +212,7 @@ function BackupProgressComponent({ websocket }) { ### Throttling Behavior - Progress updates are throttled to prevent WebSocket spam - Updates only sent when percentage increases by ≥1% -- Minimum 500ms interval between updates +- Minimum 250 ms interval between updates - No throttling for initial (0%) and final (100%/-1%) updates ### Reliability diff --git a/CHANGELOG.md b/CHANGELOG.md index f97b0e2a0..622560f53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## v1.13.4 +### Security +* Broad upstream hardening release (incl. advisory GHSA-8m75-v66f-m43j): file and archive operations are contained against symlink/hard-link escapes, ignore lists (`.pteroignore`, backup ignore) are validated and size/time-limited, decompression was rewritten, and quota/disk-usage accounting no longer follows links. +* Remote file downloads (`pull`) resolve and dial through an SSRF-checked dialer; private/internal destinations are blocked unless listed in the new `api.remote_download_allowlist`. Backup-restore downloads use the same network checks. +* SFTP gains connection limits: handshake timeout, per-user connection cap and per-IP cap on concurrently authenticating connections. +* WebSocket and HTTP API: serialized writes with deadlines, throttled sends, and stricter request/upload handling. +* Configuration updates sent by the Panel can now only change the Panel-owned values (uuid, token, API host/port/SSL, upload limit, SFTP port, debug); everything else in `config.yml` stays under the node operator's control. + +* WebSocket `activity` events now require the `activity.read` permission and `download progress` events require `file.read`; previously every websocket subscriber received them, including file paths and download URLs. +* Remote file downloads without a `Content-Length` (chunked) are now capped at 15 GB like downloads of known size. +* Backup restores from a remote location no longer log the presigned download URL (including its signature); only the host is logged. + +### Fixed +* **Deleting a server no longer deletes the local backups of every server on the node.** The server cleanup removed every file in the backup directory that looked like a backup, regardless of which server it belonged to; it also ran when the Panel sent a duplicate create request and on transfer cleanup. Local backups are now only removed when the Panel deletes a specific backup. +* A duplicate server-create request or a failed transfer no longer cancels the running backups and restores of the live server with the same UUID. +* Remote file downloads without a `Content-Length` were saved as an empty file; they are now written in full (counted against the disk limit as they stream), and a failed write is reported as failed instead of completed. +* A server transfer can no longer start while a backup of that server is running. +* Local backups stored as uncompressed `.tar` could be restored but their size/checksum lookup failed, so restore progress fell back to an estimate; both now read the actual file. +* The backup event sent when the Panel could not be notified of a successful backup now reports `checksum_type: sha256` like all other backup events. +* S3 backup uploads now really use HTTP/1.1 as intended and respect the `HTTP(S)_PROXY` environment variables. +* Backup restores no longer suspend/unsuspend the server, so a Panel-side suspension is no longer cleared by a restore. The server is protected from starting via the restoring flag instead. +* "Truncate directory" on restore now happens inside the restore after the server has stopped, rather than before the backup was even located — a missing backup or an unreadable S3 stream no longer leaves the server with deleted files. +* Restores recreate symlinks as symlinks and skip hard links, devices and other special files instead of writing them as regular files. +* Backups and restores that had to wait in the queue (more than 8 concurrent operations on a node) crashed Wings or leaked their queue slot, because the queued states were rejected by the container state check. Queued operations now work as intended. +* The server environment is no longer synced twice on start, and the "server started" console line can no longer be missed. + +### Added +* `system.sftp.max_connections_per_user` (default 32) and `system.sftp.max_authenticating_connections_per_ip` (default 16 — raise it if clients connect through a proxy). +* `api.remote_download_allowlist` — hostnames, IPs or CIDR ranges that remote file downloads may reach despite being private/internal. + +### Changed +* Wings now requires Go 1.26 to build; release binaries are built with Go 1.27.2. +* Lint CI (golangci-lint) runs on every push and pull request to `develop`. + ## v1.13.3.1 ### Changed * Fork-only re-release of v1.13.3 under a new version number — no code changes; binaries and image are rebuilt from the same sources. First release cut through the new `fork_revision` dispatch path. diff --git a/Dockerfile b/Dockerfile index f773e721d..c2af3636b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1 (Build) -FROM golang:1.24.11-alpine AS builder +FROM golang:1.27.2-alpine AS builder ARG VERSION RUN apk add --update --no-cache git make mailcap diff --git a/FORK_CHANGES.md b/FORK_CHANGES.md index 644a93a57..766ceecfd 100644 --- a/FORK_CHANGES.md +++ b/FORK_CHANGES.md @@ -4,11 +4,11 @@ This file tracks **which changes are our own** (EmeraldHost-specific) versus ups [`pterodactyl/wings`](https://github.com/pterodactyl/wings). Use it during upgrades so our customizations are **not accidentally reverted** when pulling in upstream changes. -- **Baseline for this comparison:** upstream tag **`v1.13.3`** (`6987d5e`) -- **Last reviewed:** 2026-08-14 -- **Module path:** this fork is `github.com/Rene-Roscher/wings` (upstream is - `github.com/pterodactyl/wings`). Version is injected at build time via ldflags - (`-X .../system.Version=`); `system/const.go` stays `develop` and is **not** a divergence. +- **Baseline for this comparison:** upstream tag **`v1.13.4`** (`747bc43`) +- **Last reviewed:** 2026-10-14 +- **Repo vs. module path:** the repo is `github.com/emeraldhost/wings`; the Go module path is + `github.com/Rene-Roscher/wings` (upstream is `github.com/pterodactyl/wings`). Version is injected at + build time via ldflags (`-X .../system.Version=`); `system/const.go` stays `develop` and is **not** a divergence. - **How to regenerate the picture:** ```bash git fetch upstream --tags @@ -18,120 +18,147 @@ our customizations are **not accidentally reverted** when pulling in upstream ch Diff direction: in `git diff HEAD`, a `+` line is **in our fork**, a `-` line is **upstream**. -> ⚠️ The biggest divergence by far is the **backup subsystem** (~6 000+ lines): an -> operation registry/queue, retry, WebSocket progress and a -> heavily customized restore path. Upstream merges in `server/backup*`, `router/router_server_backup.go`, -> `server/server.go` and `sftp/server.go` will almost always conflict — resolve by **keeping ours** -> and grafting upstream's functional/security changes on top (that is exactly how v1.13.1 was merged). +> ⚠️ The biggest divergence by far is the **backup subsystem** (~6 000 lines added vs v1.13.4: about +> 3 300 production code and 2 700 fork-only tests): an operation registry/queue, retry, WebSocket +> progress and a heavily customized restore path. Upstream merges in `server/backup*`, +> `router/router_server_backup.go`, `server/server.go` and `server/filesystem/archive.go` will almost +> always conflict — resolve by **keeping ours** and grafting upstream's functional/security changes on +> top (that is exactly how v1.13.1 and v1.13.4 were merged). +> `sftp/server.go` is **no longer fork code** (see §1.3/§4): on a conflict there, take upstream's version +> and re-apply only the module rename. > -> v1.13.2 and v1.13.3 were exceptions: both stayed clear of the backup subsystem and merged -> (nearly) conflict-free — see §4. The v1.13.3 conflicts were only our rewritten -> `release.yaml` (keep ours, see §1.4) and the module-renamed import block of `server/install.go`. +> v1.13.2 and v1.13.3 stayed clear of the backup subsystem and merged (nearly) conflict-free. +> v1.13.4 was a broad hardening release (153 files) with 24 conflicted files, including the whole +> backup/restore/archive path. Rows marked *(v1.13.4)* in §1 record where fork and upstream code now +> interlock; §4 lists what v1.13.4 brought in as plain upstream code. --- ## 1. Our own changes — PRESERVE on every upgrade -### 1.1 Module rename (mechanical, but must stay) +### 1.1 Module rename and repo links (mechanical, but must stay) | Path | What | Note | |------|------|------| | `go.mod` | `module github.com/pterodactyl/wings` → `github.com/Rene-Roscher/wings` | Root of the rename; every Go import of the module changes accordingly. | -| `Dockerfile`, `Makefile`, `.github/workflows/{push,release}.yaml`, `wings.go` | ldflags / `SRC_PATH` / import path use the `Rene-Roscher` module | Required so `system.Version` is injected and builds resolve. | -| ~every `*.go` file | `github.com/pterodactyl/wings/...` → `github.com/Rene-Roscher/wings/...` | **Incidental noise** — appears as conflicts on most merges but carries no behavior. Always "keep ours (Rene-Roscher)". | +| `Dockerfile`, `Makefile`, `.github/workflows/push.yaml` | ldflags `-X …/system.Version` / `SRC_PATH` use the `Rene-Roscher` module. Otherwise identical to upstream v1.13.4: on merge, take upstream and re-apply the path. | Required so `system.Version` is injected and builds resolve. The fork-only `binary.yaml` sets the same ldflags path. `release.yaml`, `binary.yaml`, `docker.yaml` are full fork rewrites, see §1.4. | +| `README.md` ("Installation" section), `cmd/root.go` (startup banner Source/License) | Point at the **repo** `github.com/emeraldhost/wings`, not the module path. Release assets are `wings_linux_{amd64,arm64}` + `SHA256SUMS` (no plain `wings` asset). | Repo URL ≠ module path: only `go.mod`/imports/ldflags use `Rene-Roscher`. | +| ~every `*.go` file | `github.com/pterodactyl/wings/...` → `github.com/Rene-Roscher/wings/...` | **Incidental noise** — appears as conflicts on most merges but carries no behavior. Always "keep ours (Rene-Roscher)". E.g. `wings.go` differs only by its `cmd` import (upstream v1.13.4 dropped `rand.Seed` there — do not re-add it). After the rename, re-run `gofmt` (§5). | ### 1.2 Backup subsystem — the largest divergence -> Upstream's backup path is small (`s.Backup(b)` / `s.RestoreBackup(b)` in a bare goroutine). +> Upstream's backup path is small (`s.Backup(b)` / `s.RestoreBackup(b, reader, truncate)` in a bare goroutine). > The fork replaced it with a queued, cancellable, progress-reporting pipeline. **Operation registry, queue, retry (server package)** | Path | What | |------|------| -| `server/backup_operations.go` | **Fork-new.** `BackupOperationRegistry` + global `GetBackupOperationRegistry()`: concurrency-limited (`maxConcurrentBackups/Restores = 8`), queueing, cancellation. `Register()` returns the 5-tuple `(*op, ctx, cancel, err, wasQueued)` consumed by the router. `Cancel/Complete/CleanupStaleOperations` block-receive the semaphore token to release slots. Background `StartBackupOperationCleanup`. | -| `server/backup.go` | `Backup()`/`RestoreBackup()` are now compat wrappers over `BackupWithContext`/`RestoreBackupWithContext` (context + 6h/4h timeouts, atomic state transitions). `BackupWithRetry(ctx,b,2)` with exponential backoff. Restore does compression auto-detection, progress, restore-stats and per-dir `MkdirAll`+`Chown`. On panel-notify failure for a **successful** backup it **does not** delete the archive (upstream does). | -| `server/server.go` | `Server.backingUp *AtomicBool` (+ init), `AtomicStateTransition` + `ApplyAtomicStateTransition()`, `CleanupForDestroy()` cancels in-flight ops + cleans orphaned backup files, `PublishActivity()`. | +| `server/backup_operations.go` | **Fork-new.** `BackupOperationRegistry` + global `GetBackupOperationRegistry()`: concurrency-limited (`maxConcurrentBackups/Restores = 8`), queueing, cancellation. `Register()` returns the 5-tuple `(*op, ctx, cancel, err, wasQueued)` consumed by the router. `Cancel/Complete/CleanupStaleOperations/CancelAllForServer` block-receive the semaphore token to release slots. Background `StartBackupOperationCleanup` (5 min ticker, drops operations older than 8 h) is launched from `cmd/root.go`. | +| `server/backup.go` | `Backup()` (own 6 h timeout) and `RestoreBackup()` (server context, no timeout) are compat wrappers over `BackupWithContext` (adds a 6 h timeout only if the ctx has no deadline) and `RestoreBackupWithContext`. The router sets the 6 h backup / 4 h restore timeouts. State transitions are atomic (`ApplyAtomicStateTransition`). `BackupWithRetry(ctx, b, 2)` (called by the router; 3 attempts) gives each attempt its own 6 h timeout and backs off quadratically (30 s·n²: 30 s, 120 s); no retry on cancel/deadline. Restore does compression auto-detection, progress and restore-stats. The S3 download size reaches the restore via the typed context key `DownloadSizeContextKey` (replaced the old `"download_size"` string key). In state `restore_queued` the restore asks `Environment.IsRunning()` instead of stopping the container. On panel-notify failure for a **successful** backup it **does not** delete the archive (upstream does). | +| `server/backup.go` restore *(v1.13.4)* | `RestoreBackup(b, reader, truncate)` / `RestoreBackupWithContext(ctx, b, reader, truncate)` carry upstream's new `truncate` flag: `TruncateRootDirectory` runs **inside** the restore, after the server has stopped and after compression detection (an undetectable S3 stream no longer wipes files first); the router no longer truncates before locating the backup. Upstream dropped the suspend/unsuspend toggling (it clobbered Panel suspensions); start protection is `SetRestoring(true)` **before** `WaitForStop` (also cancels SFTP sessions). `restoreEntry` merges both sides: fork's mode-preserving `MkdirAll`+`Chown`+`Chtimes` for directories, upstream's symlink recreation and skipping of hard links/devices/special files. | +| `server/server.go` | `Server.backingUp *AtomicBool` (+ init), `AtomicStateTransition` + `ApplyAtomicStateTransition()`, `PublishActivity()` (see §1.3). `CleanupForDestroy()` is **upstream again** since the v1.13.4 merge — no backup-op cancellation and no backup-file deletion there (see §3 resolved). | +| `router/router_server.go` `deleteServer` | Calls `GetBackupOperationRegistry().CancelAllForServer()` for the real server being deleted (moved here from `CleanupForDestroy` in the v1.13.4 merge, so throwaway duplicate-create/transfer instances with the same UUID no longer touch the live server's operations). | | `server/install.go` | `IsBackingUp()` / `SetBackingUp()`. (Note: `IsInProtectedState()` was deliberately **not** extended with `backingUp`.) | | `server/power.go`, `server/errors.go` | `HandlePowerAction()` also blocks while `IsBackingUp()`, returning new sentinel `ErrServerIsBackingUp`. | -| `environment/environment.go` (+ docker state whitelist) | New states `backup`, `restore`, `backup_queued`, `restore_queued`. | +| `environment/environment.go` (+ docker state whitelist) | New states `backup`, `restore`, `backup_queued`, `restore_queued`. The `switch` in `environment/docker/environment.go` `SetState()` must list **all four** (guarded by `environment/docker/state_test.go`). The queued states were only added to it in the v1.13.4 merge; before that, every backup/restore that had to wait for a registry slot panicked in `SetState()` and leaked its slot, and a queued restore crashed Wings. | +| `router/router_server_backup.go` goroutines | The registry cleanup `defer` (`registry.Complete` + `cancel`) is registered **directly after** a successful `Register()`, before the `wasQueued` `SetState`; both restore goroutines also have a top-level `recover`. Keep that order on merge, otherwise a panic leaks a registry slot until the 8 h stale cleanup. | +| `server/backup_restore_test.go` *(v1.13.4)* | Upstream suite; only its `restoreTestEnvironment` double gained no-op `SetState()` / `IsRunning()` because the fork's restore drives environment state. Assertions unchanged. | **Progress & activity over WebSocket** | Path | What | |------|------| -| `server/backup_progress.go` | **Fork-new.** `BackupProgressUpdate` WS payload `{backup_id,type,percentage,bytes_written,bytes_total}`, throttled tracker, S3 80/20 archive-vs-upload split. | -| `server/backup/download_progress.go` | **Fork-new.** `DownloadProgressReader` / `NewDownloadProgressReader` — S3 restore download progress. | -| `server/events.go` | New `BackupProgressEvent`, `DownloadProgressEvent`, `ActivityEvent`. **Frontend contract.** | -| `server/activity.go` | New `ActivityFile{Downloaded,Compressed,Decompressed,Chmod}`; `SaveActivity` also publishes `ActivityEvent` over WS. | +| `server/backup_progress.go` | **Fork-new.** `BackupProgressUpdate` WS payload `{backup_id,type,percentage,bytes_written?,bytes_total?}` (byte fields `omitempty`). `type` is `create` / `restore` (250 ms-throttled `SimpleProgressTracker`) or `download` (S3 restore download, published unthrottled from `router/router_server_backup.go`). Also the S3 80/20 archive-vs-upload split. | +| `server/backup/download_progress.go` | **Fork-new.** `DownloadProgressReader` / `NewDownloadProgressReader`: wraps the S3 restore download body; its callback emits `BackupProgressEvent` with `type: "download"`, **not** `DownloadProgressEvent`. | +| `internal/progress/progress.go` | Fork hook on upstream's `Progress` writer: optional `ProgressCallback func()` field (called after every `Write()`), new `AddWritten(bytes uint64)` (atomic add + callback). Wired in `server/backup.go` (`ProgressCallback = progressTracker.CheckProgress`, `restoreProgress.AddWritten`) and required by the `ProgressTracker` interface in `server/backup/backup_s3.go`. **Keep on merge** — with upstream's file, `server/backup*.go` does not compile. Caveat: the `recover` in `Write()` covers the whole function, so a panicking callback makes `Write` return `(0, nil)` without forwarding the chunk. | +| `server/events.go` | New `BackupProgressEvent` (`"backup progress"`), `DownloadProgressEvent` (`"download progress"`: remote file pull in `router/downloader`, payload `DownloadProgressUpdate {identifier,filename,directory,url,percentage,bytes_written,bytes_total,status}`), `ActivityEvent` (`"activity"`). **Frontend contract** — see `WEBSOCKET_EVENTS.md` / `BACKUP_PROGRESS_EVENTS.md`. | **Compression & checksums** | Path | What | |------|------| -| `server/backup/backup.go` | **SHA-256** checksums + `ChecksumType: "sha256"` (upstream uses **sha1**). ⚠️ **Protocol-facing** — a careless merge reverts to sha1 and breaks checksum compatibility with our Panel. `PathForLocalBackup()` helper. | +| `server/backup/backup.go` | **SHA-256** checksums + `ChecksumType: "sha256"` (upstream uses **sha1**). ⚠️ **Protocol-facing** — a careless merge reverts to sha1 and breaks checksum compatibility with our Panel. Upstream's package-level gzip-only `format` is removed: restore detects the compression via `filesystem.DetectCompressionFormat` + `CreateDecompressor` (local) or gets an already-decompressed stream (S3), then extracts with plain `archives.Tar{}`. *(v1.13.4)* Fork patch on upstream's identifier code: `validateIdentifier()` only writes `b.Uuid` when it changes, and `Details()` validates once before spawning its goroutines — a `-race` failure surfaced by upstream's new restore test (the fork's restore calls `Details()`). | | `server/backup/compression.go` | **Fork-new.** `CompressionRegistry` (gzip/tar/none) + `IsValidBackupContentType()` — used by the router content-type gate; without it the router won't compile. | -| `server/backup/backup_local.go` | `foundPath` + extension-probing `LocateLocal` (`.tar.gz/.tar`), auto-detecting `Restore`, `CleanupBackupFilesForServer`. | -| `server/backup/backup_s3.go` | Two-phase backup reuse, success-flag cleanup (failed uploads kept for retry), orphaned-part logging, upload progress (`ProgressReader`/`ProgressTracker`), custom HTTP/1.1 transport, part-retry with 100MB memory-buffer threshold / 5GB cap, `Restore()` expects an **already-decompressed** tar stream. | -| `server/filesystem/archive.go` | Archiver no longer skips directory entries → **empty directories are preserved** in archives. `createCompressor()` refactor. | +| `server/backup/backup_local.go` (+ `foundPath` in `server/backup/backup.go`) | `LocateLocal` fallback: if `.tar.gz` is missing it also looks for `.tar` and stores it in the fork-added `Backup.foundPath`, which `Backup.Path()` returns — so Remove/Restore **and** Size/Checksum/Details use the same file. Auto-detecting `Restore`. | +| `server/backup/backup_s3.go` | Two-phase backup reuse; the local archive is removed only after a successful upload (failed uploads kept for retry — *(v1.13.4)* upstream's unconditional `defer s.Remove()` was **not** taken, its error-checked removal is grafted into the success branch only); orphaned-part logging; upload progress (`ProgressReader`/`ProgressTracker`); custom `http.Transport` for part uploads (see §3); part-retry with 100 MB memory-buffer threshold / 5 GB cap; `Restore()` expects an **already-decompressed** tar stream. | +| `server/filesystem/archive.go` | Archiver no longer skips directory entries → **empty directories are preserved** in archives. `createCompressor()` refactor. *(v1.13.4)* Upstream's `compileIgnore()` + `ignoreMatchBudget`, `Readlinkat`-based symlink reading and `*[]byte` buffer pool are taken; the ignore compilation runs **before** `createCompressor()` so an invalid ignore list fails before anything is written. | | `server/filesystem/archive_restore.go` | **Fork-new.** `DetectCompressionFormat` (magic bytes) + `CreateDecompressor`, wired into restore. | -| `server/filesystem/compress_binary_test.go`, `server/backup/*_test.go`, `router/content_type_test.go`, `server/backup_*_test.go` | Fork-new regression suites for the above. Keep them passing. | -| `server/filesystem/archive_test.go` + `archive_stream_test.go` | **File layout divergence, not new behavior.** Our `archive_test.go` was replaced wholesale with tests for the fork-only `archive_restore.go`; upstream's `TestArchive_Stream` lives in `archive_stream_test.go` instead. Upstream edits to `archive_test.go` therefore land in the *wrong* file on merge — port them into `archive_stream_test.go` by hand. | +| `server/filesystem/compress_binary_test.go`, `server/backup/{content_validation,restore,restore_edge_case,s3_simple}_test.go`, `router/content_type_test.go`, `server/backup_{cleanup,progress_validation,queue,retry}_test.go`, `environment/docker/state_test.go` | Fork-new regression suites for the above. Keep them passing. (`server/backup/backup_test.go` and `server/backup_restore_test.go` are upstream suites.) | +| `server/filesystem/archive_test.go` + `archive_stream_test.go` | **File layout divergence, not new behavior.** Our `archive_test.go` holds tests for the fork-only `archive_restore.go`; upstream's `TestArchive_Stream` lives in `archive_stream_test.go` instead. Upstream edits to `archive_test.go` therefore land in the *wrong* file on merge — port them into `archive_stream_test.go` by hand. *(v1.13.4: upstream's two new ignore-pattern cases were ported there; `getFiles` `continue`s on directory entries instead of returning, because of the empty-dir preservation above.)* | **Router API (fork-only endpoints + customized handlers)** | Path | What | |------|------| -| `router/router.go` | Fork-only routes `GET /backup/operations` and `DELETE /backup/:backup/cancel`. | -| `router/router_server_backup.go` | `cancelServerBackup` + `getServerBackupOperations` (fork-only). `postServerBackup` / `postServerRestoreBackup` rewritten: 409 concurrency guards, registry queueing, timeouts, panic recovery, retry, S3 download progress, and content-type via `backup.IsValidBackupContentType` (gzip+tar) instead of upstream's gzip-only check. | +| `router/router.go` | Fork-only routes `GET /api/servers/:server/backup/operations` and `DELETE /api/servers/:server/backup/:backup/cancel`. | +| `router/router_server_backup.go` | Fork-only `cancelServerBackup` (404 when unknown/done, 403 when the op belongs to another server) and `getServerBackupOperations` (`{operations:[{id,backup_id,type,start_time}],count}`). `postServerBackup` / `postServerRestoreBackup` rewritten: 409 guards (backing up / restoring / transferring), registry queueing (`backup_queued` / `restore_queued`), 6 h / 4 h timeouts, panic recovery, `BackupWithRetry(ctx, b, 2)`. `SetRestoring` and `BackupRestoreCompletedEvent` live inside `RestoreBackupWithContext`. S3 restores wrap `res.Body` in `backup.NewDownloadProgressReader` and pass the size via `server.DownloadSizeContextKey`; only the download **host** is logged, never the presigned URL. Content-type gate is `backup.IsValidBackupContentType` (gzip family, tar **and** `application/octet-stream` via the `none` adapter; an empty `Content-Type` is accepted) instead of upstream's gzip-only check — the real format is sniffed afterwards. `deleteServerBackup` falls back to removing a leftover local S3 archive (`NewS3(client, id, "").Path()`) when `LocateLocal` misses. | +| `router/router_server_files.go` | Fork activity logging on the Panel-API file handlers: rename/delete/write/create-directory log upstream's `server:sftp.*` events (write picks `create` vs `write` from a `Stat` before the write); pull/compress/decompress/chmod log the fork-new `server:file.{downloaded,compressed,decompressed,chmod}`. All via `NewRequestActivity("", c.ClientIP())` (empty user). `postServerPullRemoteFile` calls `dl.EnableEvents()` for non-foreground pulls, and its 202 adds `filename`, `directory`, `url`. Upstream's v1.13.2 `token.Denylisted()` upload guard sits in the same file (§4). | +| `router/router_server_transfer.go` *(v1.13.4)* | Fork 409 guards (`IsBackingUp()` / `IsRestoring()`) run **before** upstream's new atomic `StartTransferring()` claim. Order matters: after it, a rejected request would leave `transferring=true` stuck. Fork patch in `server/install.go`: `StartTransferring()` also refuses while `IsBackingUp()`, so both router guards only add more specific messages. (Queued backups: see §3.) | +| `router/websocket/websocket.go` `SendJson` | Fork permission gates: `BackupCompletedEvent`, `BackupRestoreCompletedEvent` (prefix match) **and** `BackupProgressEvent` need `backup.read`; the fork-only `activity` event needs `activity.read` and `download progress` needs `file.read` (added in the v1.13.4 merge — before that they reached every websocket subscriber). Rewritten in v1.13.4 onto upstream's single `validJwt()` token; everything else (write lock + deadline, `SendThrottled`, `SendClose`, `OnAuthenticated`) is upstream. | +| `router/downloader/downloader.go` | Fork redirect-following loop (upstream follows **no** redirects; the fork follows up to 10 hops, each through the SSRF-checked dialer). A missing `Content-Length` (chunked) is allowed where upstream rejects it; `maxDownloadSize` (15 GB) / `ErrFileTooLarge` is enforced for known sizes up front and for unknown sizes through `maxSizeReader`; unknown-size bodies are written with upstream's `Filesystem().WriteFrom` (quota per write, file removed on failure) because `Write` copies nothing for a size of -1. Strict `sanitizeFilename` allowlist on header/user/URL filenames (stricter than upstream: rejects spaces and dotfiles). `download progress` events are opt-in via `EnableEvents()` (background pulls only), throttled to 250 ms, with a final `completed`/`failed` event. *(v1.13.4)* Upstream's dialer/`RemoteDownloadAllowlist` SSRF hardening is taken; its `%w` error wrapping is grafted into the fork loop and the `Content-Disposition` name is reduced with `filepath.Base(filepath.Clean("/"+v))` before `sanitizeFilename`. | -### 1.3 SFTP activity streaming +### 1.3 Activity streaming over WebSocket (SFTP + HTTP actions) | Path | What | |------|------| -| `sftp/event.go`, `sftp/handler.go` | `EventPublisher` interface + `publisher` on the event handler → SFTP file actions streamed to the panel via `Server.PublishActivity` (in addition to DB persistence). | +| `server/activity.go` | New events `server:file.{downloaded,compressed,decompressed,chmod}` (`ActivityFile*`, logged by `router/router_server_files.go`). `SaveActivity` also publishes `ActivityEvent` (`{event,user,metadata}`) in a goroutine with `recover`, so every activity logged through it reaches WS clients. | +| `sftp/event.go`, `sftp/handler.go`, `server/server.go` `PublishActivity()` | `EventPublisher` interface + `publisher` field on the SFTP event handler (`publisher: srv` in `NewHandler`). After the DB insert, `Log()` publishes `{event,user,metadata}` asynchronously (goroutine + `recover`) via `Server.PublishActivity` → `ActivityEvent` on the server event bus → WebSocket clients. The Panel itself still receives activity only through the DB/cron path. `PublishActivity` ignores its `event` argument. `handler.go` also carries upstream's v1.13.3 setstat hardening (§4). | +| `server/backup.go` `BackupWithRetry` | Once every retry has failed, publishes an `ActivityEvent` with a **different** payload: `{event:"backup_failed",timestamp,backup_id,server_id,error,total_attempts,message}`. | > The fork's previous `SmartSecurityProtector` SFTP brute-force/IP-reputation system > (and its `sftp.security.*` config) was **removed** — `sftp/server.go` now matches upstream -> (vanilla SFTP auth) apart from the module rename. SFTP abuse protection is left to the -> network layer (firewall / fail2ban) / the Panel. +> (vanilla SFTP auth plus upstream v1.13.4's connection limits) apart from the module rename. -### 1.4 Repo config +### 1.4 Repo config & docs | Path | What | |------|------| -| `.gitignore` | Fork-added `.claude-flow/`, `.hive-mind/`, `CLAUDE.md`. Upstream will never add these — keep on merge. | -| `Makefile`, `Dockerfile` | Our build settings (with the renamed module path). | -| `.github/workflows/{release,binary,docker}.yaml` | **Fork-specific release pipeline — always keep ours.** Upstream releases by hand: a human pushes a `v*` tag, `release.yaml` cuts a draft, a human publishes it. We release automatically from `develop` instead, and the version is derived from the newest **upstream** tag that is an ancestor of `develop` — so our releases always carry the upstream version number. Fork-only releases between upstream versions append a fourth component (`v1.13.3.1`), cut by dispatching `release.yaml` with `fork_revision`; the automatic path never does this. Upstream's `release.yaml` has diverged beyond recognition; do not merge it (v1.13.3's `c57c519` CDN-manifest notification was deliberately dropped — it notifies pterodactyl's own CDN repo). See the header comment in `release.yaml` for the full flow and recovery steps. | +| `.gitignore` | Fork-added `.claude-flow/`, `.hive-mind/`, `CLAUDE.md` (local AI-tooling files; never committed) and `/wings-*`, `/dist/` (local build artifacts). Upstream will never add these — keep on merge. | +| `.github/workflows/{release,binary,docker}.yaml` | **Fork-specific release pipeline — always keep ours.** Upstream releases by hand: a human pushes a `v*` tag, `release.yaml` cuts a draft, a human publishes it. We release automatically from `develop` instead, and the version is derived from the newest **upstream** tag that is an ancestor of `develop` — so our releases always carry the upstream version number. Fork-only releases between upstream versions append a fourth component (`v1.13.3.1`), cut by dispatching `release.yaml` with `fork_revision`; the automatic path never does this. Upstream's `release.yaml` has diverged beyond recognition; do not merge it (v1.13.3's `c57c519` CDN-manifest notification was deliberately dropped — it notifies pterodactyl's own CDN repo; upstream's `git config` bot identity is gone too, our pipeline makes no git commits). See the header comment in `release.yaml` for the full flow and recovery steps. Upstream action/toolchain bumps are still grafted by hand where they apply to our jobs (v1.13.4: Go 1.27.2 + `setup-go` v7 in `binary.yaml` — required since `go.mod` says `go 1.26` — `docker/login-action` v4 in `release.yaml`, `docker/build-push-action` v7 in `docker.yaml`). | +| `CHANGELOG.md` | **Fork-maintained.** Upstream's file stops at `v1.12.3`; all `v1.13.x` entries (incl. fork-only `v1.13.3.1`) are written by us and double as release notes (the release gate refuses to publish without the section). Write each new version's entry by hand from upstream's commits; on a conflict keep ours. | +| `README.md` ("Fork Features" block) | Fork-only summary of the backup/restore pipeline and WS events, between upstream's intro and "Pterodactyl v2 is coming soon". On merge keep our block plus the Installation section, take upstream's text elsewhere. | +| `BACKUP_PROGRESS_EVENTS.md`, `WEBSOCKET_EVENTS.md` | **Fork-only** frontend-contract docs for the WS events. Upstream has no such files. Update them whenever an event payload, permission or throttle changes. | --- -## 2. Config divergences — confirm whether intentional +## 2. Hardcoded value divergences — confirm whether intentional -These sit on **different** values/fields than upstream; they will re-appear in a generated -`config.yml`. Review on upgrade. +These are fork-chosen **hardcoded constants/literals in Go code** (not `config.yml` options — they never +appear in a generated `config.yml`); changing them per node needs a code change. Review on upgrade. -| Path | Fork value / field | Note | -|------|--------------------|------| -| `server/backup_operations.go` | `maxConcurrentBackups/Restores = 8`; cleanup ticker 5 min / op TTL 8 h; backup 6 h / restore 4 h timeouts | Fork-chosen capacity/timeouts. | -| `server/backup_progress.go` | 250 ms throttle; S3 80/20 split; 1 MB chunking | Determines WS emission rate / S3 percentage curve. | +| Path | Fork value | Note | +|------|------------|------| +| `server/backup_operations.go` | `maxConcurrentBackups/Restores = 8`; stale-op cleanup ticker 5 min, op TTL 8 h | Fork-chosen capacity. | +| `router/router_server_backup.go`, `server/backup.go` | Backup timeout 6 h (router; `BackupWithContext` only without a deadline; per `BackupWithRetry` attempt; `Backup()`). Restore timeout 4 h, set only in the router. Retries: 2 (30 s, 120 s backoff). Upstream has none of these. | Fork-chosen timeouts. Retries cannot exceed the outer 6 h deadline. | +| `server/backup_progress.go` (+ `server/backup/backup_s3.go`, `router/downloader/downloader.go`) | 250 ms WS throttle; S3 80/20 archive/upload split; byte mode (total unknown) counts whole MB; S3 upload reader caps reads at 1 MB (`maxChunk`) | Sets the WS event rate / S3 percentage curve. The 250 ms value is duplicated in three files — change them together. | | `server/backup/backup_s3.go` | per-part upload `Content-Type: application/octet-stream` (upstream `application/x-gzip`) | Fork choice. Verify the Panel/S3 presigned flow tolerates it. | --- ## 3. Known issues in our own fork code (tech debt) -Every item below was **verified fork-only** (`git grep` against upstream `e771816` returns -zero hits) — upstream wings does **not** do these, so they are **our** code/behavior, not -inherited upstream defaults that can be ignored. These are not "divergences to preserve" so -much as bugs/concerns in our own additions, worth fixing rather than defending on upgrade. +Every open item below was **verified fork-only** against upstream **`v1.13.4`** — the code involved does +not exist upstream. These are bugs/concerns in our own additions; fix them rather than defend them on upgrade. | Area | Concern (all fork-only) | |------|---------| -| **Committed binaries** | `wings-debug`, `wings-fixed`, `wings-test` (~41 MB each) and `dist/wings_test` (~28 MB) are committed build artifacts (~150 MB total), not gitignored. Repo bloat / accidental commits. | -| **Backup cleanup scope** | `cleanupBackupFiles` (server.go) and `CleanupBackupFilesForServer` (backup_local.go) match backup files by **filename pattern only** and do **not** filter by the server's ID. Since the backup directory is shared, deleting one server can remove **other** servers' local backups. | -| **`checksum_type` label** | `server/backup.go` emits `"sha256"` in most events but still `"sha1"` in the panel-notify-failure success branch. Reconcile the labels (actual algorithm is sha256). | -| **`validateBackupContent`** | Fails a backup on any server-vs-archive file/dir count mismatch (can race a live server writing files), and computes a full SHA-256 over the **entire server tree and backup file** purely for a debug log line (perf cost on large servers). | +| **Queued backup/restore guards** | `backingUp`/`restoring` are set only **after** the registry slot is acquired, and `Register()` blocks until then without making the op visible anywhere. So a backup/restore that waits in the queue (more than 8 concurrent on the node) passes every 409 guard (backup, restore, transfer, power) until it actually starts. Fix: a per-server pending counter set synchronously in the handler before the 202, released when the op ends. | + +**Resolved in the v1.13.4 merge** (kept briefly as a reminder to re-check after merges that touch these files): + +- **Node-wide backup wipe:** `CleanupForDestroy()` ran `cleanupBackupFiles()`, which deleted **every** `<36-char-id>.tar.gz/.tar/.gz` in the shared backup directory — all local backups of all servers on the node — on every server deletion, 409 duplicate create, duplicate/failed incoming transfer and boot-time duplicate. `cleanupBackupFiles` and the dead `CleanupBackupFilesForServer` are removed; local backups are only deleted by the Panel's per-backup delete, like upstream. +- **Duplicate instances cancelled the live server's operations:** `CancelAllForServer` moved out of `CleanupForDestroy()` into `router/router_server.go` `deleteServer`, so throwaway duplicate-create/transfer instances with the same UUID no longer cancel the live server's backups/restores. +- **Uncapped chunked downloads:** responses without `Content-Length` are now capped at `maxDownloadSize` (15 GB) via `maxSizeReader` (`ErrFileTooLarge`). They also used to land as an **empty file**: `Filesystem().Write` copies nothing for a size of -1, so they now go through `WriteFrom`. A failed write now also sets the outer `err`, so the final `download progress` event reports `failed` instead of `completed`. +- **Presigned URL in logs:** the backup restore logged the full presigned `download_url` (incl. S3 signature) at Info level; now only the host is logged. +- **Ungated WS events:** `activity` and `download progress` reached every websocket subscriber (file paths, pull URLs, `backup_failed` errors); `activity` now requires `activity.read`, `download progress` requires `file.read`. +- **Dead validation code:** `validateBackupContent` / `validateBackupIntegrity` (+ their count/hash helpers) were unreachable and are removed. +- **Committed binaries:** `wings-debug`, `wings-fixed`, `wings-test` and `dist/wings_test` (~153 MB) are untracked; `.gitignore` now has `/wings-*` and `/dist/`, and `binary.yaml` runs `mkdir -p dist` (the directory used to exist only because of the committed binary). The blobs stay in history until a rewrite. +- **`checksum_type` label:** the kept-backup/panel-notify-failure event now says `"sha256"` like all others. +- **Transfer during a running backup:** `StartTransferring()` now also refuses while `IsBackingUp()`, so the check is part of the atomic claim and not only the router guard. +- **S3 upload transport:** `newS3FileUploader` really forces HTTP/1.1 now (empty non-nil `TLSNextProto`; `ForceAttemptHTTP2: false` alone did not) and honours `Proxy: http.ProxyFromEnvironment` again. +- **`LocateLocal` `.tar` fallback:** `foundPath` moved from `LocalBackup` into `Backup`, so `Size()/Checksum()/Details()` read the found `.tar` too (they used the `.tar.gz` path and failed). `PathForLocalBackup()` (no callers) removed. Covered by `server/backup/locate_local_test.go`. +- **Queued states crashed the router:** see the `environment/environment.go` row in §1.2. --- @@ -142,23 +169,39 @@ fork changes risks duplicating or mis-merging them on the next upgrade. | Path | Reality | |------|---------| -| `config/config.go` → `ResolveToken(remote bool)`; `remote/http.go` → `Client.SetCredentials()` + mutex-guarded credentials; `router/router_system.go` → token re-resolve/empty-token guard/credential rotation in `postUpdateConfiguration` | **Upstream v1.13.3** master-key-rotation cluster (`07ce5fe`, `392e52c`, `3e6c2c9`): Panel-sent master key resets now propagate to the running daemon, with remote token values barred from `file://`/`$VAR` indirection and checked against `WINGS_TOKEN_ID`/`WINGS_TOKEN` overrides. Fork edit: module rename only. | -| `sftp/handler.go` → `setstatMode()` + `sftpAttributeExtended` rejection | **Upstream v1.13.3** (`da1a216`) hardening against unbounded allocations from crafted setstat packets. The fork's `publisher` wiring in `NewHandler` sits in the same file — both must survive a merge. | -| `environment/docker/cgroup_burst.go` (+ test), `applyCpuBurst`/`clearCpuBurst`/`SetCpuBurst` call sites in `container.go`/`power.go`/`server/install.go`, `config_docker.go` → `CpuPeriod`/`CpuBurst`/`CpuShares`, `environment/settings.go` quota math | **Upstream v1.13.3** CPU allocation work (`2cc8a10`, `6987d5e`). Not fork code — only the imports in the new files were renamed to `Rene-Roscher` (they arrive `pterodactyl` on every upstream merge; grep for leaks). | -| `config/config_token_test.go`, `remote/http_test.go`, `router/router_system_test.go`, `environment/docker/cgroup_burst_test.go`, `config/config_docker_test.go`, upstream additions in `sftp/handler_test.go` | **Upstream v1.13.3** suites, module rename only. Not fork suites. | -| `router/tokens/websocket.go` → `isDenylisted()`, and `Denylisted()` on `FilePayload` / `BackupPayload` / `UploadPayload` (+ their new `user_uuid` claim) | **Upstream v1.13.2** (`28af6dd`, "update token validation"). Revocation checking was extracted out of `WebsocketPayload.Denylisted()` into a shared `isDenylisted()` and applied to the backup-download, file-download and file-upload one-time tokens, which previously only checked `IsUniqueRequest()`/scope. Also tightened `Before(t)` → `!After(t)`, so a token issued in the same second as the revocation is now denied. All four files are byte-identical to upstream — **keep them that way**. | -| `router/tokens/denylist_test.go` | **Upstream v1.13.2**, unmodified. Covers the four payload types above. Not a fork suite. | -| `router/router_download.go`, `router/router_server_files.go` → the `token.Denylisted() \|\|` guards | **Upstream v1.13.2** call sites. The surrounding files *are* fork-modified (module rename + activity logging), so these three one-liners are easy to lose in a conflict resolution — check they survive. | -| `router/router_server_backup.go` SSRF cluster — `backupRestoreHttpClient`, `validateBackupDownloadUrl`, `parseBackupUuid`, `isBlockedBackupRestoreIP`, `isExplicitlyBlockedBackupRestoreIP`, `isAllowedBackupRestoreDestination`, `isSupportedBackupRestoreContentType`, `blockedBackupRestorePrefixes`, `backupDownloadError` | **Upstream v1.13.1** backup-restore SSRF hardening. The **only** fork edit in this cluster: the restore handler calls `backup.IsValidBackupContentType` instead of `isSupportedBackupRestoreContentType` (the latter is retained only for upstream parity + its test). | -| `config/config.go` → `Backups.RestoreHostAllowlist` | **Upstream v1.13.1.** Pairs with the SSRF allowlist above. Not a fork field. | -| `server/backup/backup.go` → `validateIdentifier()` / `normalizedIdentifier()` (+ `Path()` `path.Base` fallback) | **Upstream v1.13.1** UUID hardening. The fork uses them unchanged. | +| `internal/network/*` (`LimitListener`, address/allowlist helpers), SFTP connection limits in `sftp/server.go`, websocket write lock/deadline/throttling, `server/filesystem/ignore.go` (`ValidateIgnore`, ignore limits), `server/mounts.go` mount validation, `system/locker.go` `ErrLockerDestroyed`, `internal/cron/delivery.go`, `internal/ufs` link-containment work, `server/filesystem/compress.go` rewrite (and removal of `archiverext/`), `server/listeners.go`, `router/websocket/listeners.go`, `config/config.go` `PanelConfiguration` | **Upstream v1.13.4** hardening, module rename only. `sftp/server.go`, `server/listeners.go`, `router/websocket/listeners.go`, `server/filesystem/{compress,errors}.go` are byte-identical to upstream apart from imports — keep them that way. The ~60 new `*_test.go` files of v1.13.4 are upstream suites. (`internal/progress/progress.go` is the exception under `internal/` — fork-modified, see §1.2.) | +| `.github/workflows/lint.yaml`, `.golangci.yml` | **Upstream v1.13.4**, adopted unchanged: golangci-lint v2 (standard linters + `gosec`, `gofmt` formatter) on push/PR to `develop`. The fork adds **no** `exclusions.rules`; fork findings are fixed in code, and deliberate exceptions carry a targeted `//nolint: // ` comment (diff `git grep -n nolint` against upstream to list them). Keep it that way: no disabling linters or the formatter for fork code. | +| `config/config.go` → `ResolveToken(remote bool)`; `remote/http.go` → `Client.SetCredentials()` + mutex-guarded credentials; `router/router_system.go` → token re-resolve/empty-token guard/credential rotation in `postUpdateConfiguration` | **Upstream v1.13.3** master-key-rotation cluster (`07ce5fe`, `392e52c`, `3e6c2c9`). Fork edit: module rename only. | +| `sftp/handler.go` → `setstatMode()` + `sftpAttributeExtended` rejection | **Upstream v1.13.3** (`da1a216`) hardening. The fork's `publisher` wiring in `NewHandler` sits in the same file — both must survive a merge. | +| `environment/docker/cgroup_burst.go` (+ test), `applyCpuBurst`/`clearCpuBurst`/`SetCpuBurst` call sites in `container.go`/`power.go`/`server/install.go`, `config_docker.go` → `CpuPeriod`/`CpuBurst`/`CpuShares`, `environment/settings.go` quota math | **Upstream v1.13.3** CPU allocation work (`2cc8a10`, `6987d5e`). Module rename only. | +| `config/config_token_test.go`, `remote/http_test.go`, `router/router_system_test.go`, `environment/docker/cgroup_burst_test.go`, `config/config_docker_test.go`, upstream additions in `sftp/handler_test.go` | **Upstream v1.13.3** suites, module rename only. | +| `router/tokens/websocket.go` → `isDenylisted()`, and `Denylisted()` on `FilePayload` / `BackupPayload` / `UploadPayload` (+ their `user_uuid` claim) | **Upstream v1.13.2** (`28af6dd`). Revocation checking applied to the backup-download, file-download and file-upload one-time tokens; `Before(t)` → `!After(t)`. All four files are byte-identical to upstream — **keep them that way**. | +| `router/tokens/denylist_test.go` | **Upstream v1.13.2**, unmodified. | +| `router/router_download.go`, `router/router_server_files.go` → the `token.Denylisted() \|\|` guards | **Upstream v1.13.2** call sites: 2× `router_download.go` (backup + file download; the file is upstream apart from the module rename) and 1× `router_server_files.go` `postServerUploadFiles` (that file is fork-modified, §1.2 — the guard most likely to get lost in a conflict). Check all three survive. | +| `router/router_server_backup.go` SSRF cluster — `backupRestoreHttpClient`, `validateBackupDownloadUrl`, `parseBackupUuid`, `isBlockedBackupRestoreIP`, `isAllowedBackupRestoreDestination`, `isSupportedBackupRestoreContentType`, `backupDownloadError` | **Upstream v1.13.1** backup-restore SSRF hardening; since **v1.13.4** the IP checks delegate to `internal/network` (`IsInternal` / `IsAllowed`) and `blockedBackupRestorePrefixes` / `isExplicitlyBlockedBackupRestoreIP` are gone upstream — do not resurrect them. The **only** fork edit in this cluster: the restore handler calls `backup.IsValidBackupContentType` instead of `isSupportedBackupRestoreContentType` (retained for upstream parity + its test). | +| `config/config.go` → `Backups.RestoreHostAllowlist` | **Upstream v1.13.1.** Not a fork field. | +| `server/backup/backup.go` → `validateIdentifier()` / `normalizedIdentifier()` (+ `Path()` `path.Base` fallback) | **Upstream v1.13.1** UUID hardening — not fork code. `normalizedIdentifier()` and the `Path()` fallback are unchanged; `validateIdentifier()` carries the small fork race fix documented in §1.2 *(v1.13.4)* — keep it on merge. | | `system/const.go` | Byte-identical to upstream (`Version = "develop"`). | -| `.github/FUNDING.yaml` (`github: [pterodactyl]`) | **Upstream default, unchanged** (`git diff 28af6dd HEAD` is empty). Stale for a fork (sponsorship points at upstream) but NOT our change — clean it up if desired, don't track it as a divergence. | -| `.github/workflows/release.yaml` release-bot identity (`ci@pterodactyl.io` / `Pterodactyl CI`) | **Upstream default, unchanged.** Upstream's release.yaml already sets this identity. Not our divergence. | +| `.github/FUNDING.yaml` (`github: [pterodactyl]`) | **Upstream default, unchanged** (`git diff v1.13.4 HEAD -- .github/FUNDING.yaml` is empty). Stale for a fork but NOT our change. | > ⚠️ **Panel coupling introduced by v1.13.2.** `isDenylisted()` **fails closed**: a token with no -> `iat`, no `server_uuid` or no `user_uuid` is rejected outright. The `user_uuid` claim is new in -> v1.13.2, so backup downloads, file downloads and file uploads only work against a Panel that -> puts `user_uuid` into those JWTs. Against an older Panel every such request returns -> `404 "The requested resource was not found on this server."` — deploy Panel **before** Wings, -> and if downloads/uploads start 404-ing after a Wings upgrade, this is the first thing to check. +> `iat`, no `server_uuid` or no `user_uuid` is rejected outright. Backup downloads, file downloads and +> file uploads only work against a Panel that puts `user_uuid` into those JWTs; against an older Panel +> they return `404`. Deploy Panel **before** Wings. +> +> ⚠️ **Panel coupling introduced by v1.13.4.** `postUpdateConfiguration` only applies the +> `PanelConfiguration` fields (debug, uuid, token id/token, API host/port/SSL/upload limit, SFTP port); +> anything else the Panel sends is ignored and stays under the node operator's control. + +--- + +## 5. Upgrade merge checklist + +1. `git fetch upstream --tags`, then `git merge ` on an `upgrade-` branch; resolve per §1 (backup/archiver areas: keep ours, graft upstream fixes). +2. Rename leaked imports: `github.com/pterodactyl/wings` → `github.com/Rene-Roscher/wings` in every `.go` file (new upstream files arrive with the old path). Verify with `git grep -n 'github.com/pterodactyl/wings' -- '*.go' go.mod` — must print nothing. (A looser `pterodactyl/wings` grep also hits the log path `/var/log/pterodactyl/wings.log` in `cmd/diagnostics.go`; that stays.) +3. **Run `gofmt -w` (or `goimports -w`) on the renamed files.** The rename changes import sort order (`github.com/Rene-Roscher/...` sorts before `github.com/apex`/`gin-gonic`/...). On Windows, check against an LF export of the index (`git -c core.autocrlf=false checkout-index -a --prefix=/` then `gofmt -l `), since the CRLF working tree reports every file. +4. `GOOS=linux go build ./...` and `GOOS=linux go vet ./...` (both work from Windows); `go test -race ./...` must run on Linux — a Linux host, WSL, or a `golang:` Docker container with the repo mounted. +5. If the merge touched `environment/docker/environment.go`, `environment/environment.go` or `router/router_server_backup.go`: run `go test ./environment/docker -run SetState` and check by eye that in all three `Register()` call sites the `registry.Complete`/`cancel` `defer` comes before `if wasQueued { … SetState … }`, and that both restore goroutines open with a top-level `recover`. +6. `make lint` (= `GOOS=linux golangci-lint run ./...`) with the version pinned in `lint.yaml` (currently v2.11.4) must report 0 issues **before** opening the upgrade PR. Always lint with `GOOS=linux`: a bare run on Windows/macOS skips the `_linux`/`_unix` files CI checks. +7. Update this file (baseline tag; §4 for adopted upstream code; remove fixed items from §3, moving any lasting invariant into §1) and write the `CHANGELOG.md` section. Re-verify every "byte-identical" / "unchanged" / "module rename only" claim in §4 against the new tag — for each path this must print nothing: + `git diff --ignore-cr-at-eol HEAD -- | grep -E '^[+-]' | grep -vE '^(\+\+\+|---) ' | grep -vE 'Rene-Roscher|pterodactyl/wings'` diff --git a/Makefile b/Makefile index 9c7ad9824..23a6d733e 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,11 @@ rmdebug: cross-build: clean build compress +# Wings only targets Linux, so lint as Linux to cover the _linux/_unix files. +lint: + GOOS=linux golangci-lint run ./... + clean: rm -rf build/wings_* -.PHONY: all build compress clean +.PHONY: all build compress clean lint \ No newline at end of file diff --git a/README.md b/README.md index e12252e5c..1bb8739e5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -[![Logo Image](https://cdn.pterodactyl.io/logos/new/pterodactyl_logo.png)](https://pterodactyl.io) +

+ + + + Pterodactyl + + +

![Discord](https://img.shields.io/discord/122900397965705216?label=Discord&logo=Discord&logoColor=white) ![GitHub Releases](https://img.shields.io/github/downloads/pterodactyl/wings/latest/total) @@ -13,122 +20,52 @@ instances, fetch server logs, generate backups, and control all aspects of the s In addition, Wings ships with a built-in SFTP server allowing your system to remain free of Pterodactyl specific dependencies, and allowing users to authenticate with the same credentials they would normally use to access the Panel. -## Fork Features (Rene-Roscher/wings) - -This fork includes significant enhancements over the original Pterodactyl/Wings, focusing on real-time progress tracking, improved S3 support, and production-grade reliability. - -### 🎯 Key Improvements Over Original Wings - -| Feature | Original Wings | This Fork | -|---------|---------------|-----------| -| **Backup Progress** | No real-time updates | Live WebSocket events with 250ms updates | -| **Restore Progress** | Silent operation | Real-time file-by-file tracking | -| **S3 Upload Progress** | No feedback during upload | 80/20 split (80% archive, 20% upload) with continuous updates | -| **S3 Download Progress** | Silent download | Live download progress with MB tracking | -| **Server States** | Limited states | Added `backup` and `restore` states | -| **State Management** | Basic transitions | Atomic state transitions with proper cleanup | -| **Event Completion** | Missing events | Guaranteed `BackupRestoreCompletedEvent` | -| **Memory Management** | Variable | Optimized streaming with zero allocation in hot paths | -| **Goroutine Safety** | Basic | Full lifecycle management with context cancellation | -| **Progress Accuracy** | N/A | Intelligent size estimation with fallback | - -### Real-time Backup & Restore Progress Tracking - -This fork provides ultra-responsive real-time progress tracking for backup creation and restoration operations via WebSocket events: - -#### Backup Progress Events -- **Live percentage tracking** with intelligent size estimation -- **Real-time byte counters** showing data processed -- **250ms update intervals** for maximum responsiveness without spam -- **Smart throttling** prevents WebSocket overload while maintaining live feel -- **Fallback to bytes-only mode** when size estimation unavailable -- **S3 80/20 split**: 80% progress during archive creation, 20% during upload - -#### Restore Progress Events -- **File-by-file progress tracking** with real-time updates -- **Real-time restoration status** showing files being processed -- **Intelligent progress calculation** based on backup file size estimation -- **Ultra-live updates** for immediate user feedback -- **S3 download progress** with MB tracking and percentage updates - -#### Server State Management -- **New server states**: `backup` and `restore` for clear operation visibility -- **Smart state restoration** automatically detects actual container state after operations -- **WebSocket state events** keep frontend synchronized with server status -- **Robust state handling** prevents race conditions during concurrent operations - -#### Event Payloads -All progress events include comprehensive information: -```json -{ - "backup_id": "47363ce7-d70a-430e-8e75-6dc87c8d016d", - "type": "create|restore", - "percentage": 45, - "bytes_written": 1048576, - "bytes_total": 2097152 -} -``` - -#### Performance & Reliability -- **Zero performance impact** on backup/restore operations (<0.00001% CPU overhead) -- **Ultra-lightweight tracking** with atomic operations only -- **Memory-safe streaming** with 32KB buffers, no full-file buffering -- **Goroutine lifecycle management** with context cancellation and WaitGroup timeouts -- **Panic recovery** in all async operations -- **Smart size estimation** uses cached disk usage with 5s timeout fallback -- **Graceful degradation** maintains functionality even with estimation failures - -### Production-Grade Improvements - -#### Critical Bug Fixes -- **Fixed defer execution order** ensuring events are sent before state cleanup -- **Resolved WaitGroup deadlocks** with 100ms timeout protection -- **Fixed error capture** in defer statements (captured at execution time, not declaration) -- **Eliminated goroutine leaks** with proper context cancellation -- **Fixed S3 upload progress** with chunked reading and optimized HTTP transport - -#### S3-Specific Enhancements -- **80/20 progress split** for accurate progress during archive (0-80%) and upload (80-100%) -- **Continuous upload feedback** eliminating 12+ second silent periods -- **Download progress tracking** with real-time MB updates -- **Optimized HTTP/1.1 transport** for reliable streaming uploads -- **2-hour upload timeout** for large backups -- **Multipart upload support** with proper cleanup on failure - -### Enhanced Activity Logging - -- **Complete file operation tracking** for SFTP, HTTP API, and console commands -- **Real-time WebSocket events** for all file system changes -- **Comprehensive activity metadata** including file paths, users, and operation types -- **Automatic event publishing** with panic recovery for maximum reliability - -### Technical Implementation Details - -#### WebSocket Events -- `backup_progress`: Real-time backup creation progress -- `restore_progress`: Real-time restore operation progress -- `backup_restore_completed`: Completion notification with success status -- `server_status`: State changes including new `backup` and `restore` states - -#### Code Quality -- **CLAUDE.md guidelines**: Comprehensive development standards -- **Performance requirements**: <0.00001% CPU overhead mandate -- **Testing coverage**: Race condition tests, memory leak detection -- **Production hardening**: 10GB+ backup testing, concurrent operation safety +## Fork Features (emeraldhost/wings) + +This fork extends Pterodactyl Wings with a queued, cancellable backup/restore pipeline, live progress over the +WebSocket and activity streaming. The full list of divergences from upstream is kept in +[`FORK_CHANGES.md`](FORK_CHANGES.md); the WebSocket payloads are documented in +[`WEBSOCKET_EVENTS.md`](WEBSOCKET_EVENTS.md) and [`BACKUP_PROGRESS_EVENTS.md`](BACKUP_PROGRESS_EVENTS.md). + +| Feature | Upstream Wings | This fork | +|---------|----------------|-----------| +| **Backup/restore concurrency** | Unbounded goroutines | Operation registry: max 8 backups + 8 restores per node, the rest queued; cancellable (`DELETE /api/servers/:server/backup/:backup/cancel`, `GET /api/servers/:server/backup/operations`) | +| **Backup progress** | None | `backup progress` events (throttled to 250 ms); S3 backups report 0–80 % for the archive and 80–100 % for the upload | +| **Restore progress** | None | `backup progress` events for the restore, plus `type: "download"` progress while an S3 backup is downloaded | +| **Server states** | `offline`, `starting`, `running`, `stopping` | Adds `backup`, `restore`, `backup_queued`, `restore_queued`; power actions are blocked while a backup runs | +| **Backup checksums** | SHA-1 | SHA-256 (`checksum_type: "sha256"`) | +| **Backup formats** | gzip only | gzip and plain tar, detected by magic bytes on restore; empty directories are preserved | +| **Failed backups** | No retry | Up to two automatic retries (30 s, 120 s backoff); failed S3 uploads keep the local archive for the retry | +| **Remote file pulls** | No redirects, no progress | Follows up to 10 redirects (each SSRF-checked), 15 GB cap, `download progress` events | +| **Activity** | Stored for the Panel only | Additionally pushed live as `activity` WebSocket events (SFTP and HTTP file actions) | + +### WebSocket events added by this fork +- `backup progress`: `{backup_id, type, percentage, bytes_written, bytes_total}` with `type` = `create` | `restore` | `download` +- `download progress`: progress of a remote file pull +- `activity`: file/SFTP activity (`{event, user, metadata}`) +- `status`: now also carries `backup`, `restore`, `backup_queued` and `restore_queued` + +Upstream's `backup completed` and `backup restore completed` events are still sent when an operation finishes. ## Installation This fork is a drop-in replacement for the original Wings. Simply replace your Wings binary with this version: ```bash -# Build from source -go build -o wings cmd/root.go +# Download the pre-built binary (linux amd64 / arm64) +curl -L -o /usr/local/bin/wings "https://github.com/emeraldhost/wings/releases/latest/download/wings_linux_$([[ "$(uname -m)" == "x86_64" ]] && echo "amd64" || echo "arm64")" +chmod u+x /usr/local/bin/wings -# Or download pre-built binary (if available) -wget https://github.com/Rene-Roscher/wings/releases/latest/download/wings -chmod +x wings +# Or build from source +go build -o wings . ``` +## Pterodactyl v2 is coming soon + +Wings is stable and remains the daemon for Pterodactyl: the same Wings is used with the upcoming Panel v2. + +[![Watch: Introducing Pterodactyl v2](.github/assets/v2-video.jpg)](https://www.youtube.com/watch?v=1ER4kv0jVEU) + ## Sponsors I would like to extend my sincere thanks to the following sponsors for helping fund Pterodactyl's development. @@ -136,18 +73,19 @@ I would like to extend my sincere thanks to the following sponsors for helping f | Company | About | |-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [**Infraly, LLC**](https://infraly.co/) | Infraly is an infrastructure company powering the next generation of online services. Through their brands, Infraly delivers cutting-edge solutions across multiple markets. Their vertically integrated approach provides unmatched performance, scalability, and reliability, giving our customers full control. | -| [**Hosturly**](https://hosturly.com/) | Hosturly is an enterprise hosting provider. They provide cost-effective, high-performance, and reliable services, including VPS, Web, Dedicated, and Colocation. | -| [**Physgun**](https://physgun.com/) | Physgun is a game server hosting provider. Most providers rent rack space and rebrand a panel. At Physgun, they engineer the performance, write the features, and staff the support. Physgun truly is game hosting perfected! | -| [**WISP**](https://wisp.gg/) | WISP is an industry-leading SaaS platform for game server management, designed for hosting companies, gaming organizations, and enthusiasts. WISP combines modern, intuitive interfaces with powerful tools, making server deployment and administration seamless, scalable, and efficient. | | [**Buildurly**](https://buildurly.com/) | Buildurly is a hardware procurement company. They deliver tailored, enterprise-grade hardware solutions designed around your unique needs. From sourcing to delivery, Buildurly's white-glove service ensures a seamless, worry-free, professional experience. | +| [**Hosturly**](https://hosturly.com/) | Hosturly is an enterprise hosting provider. They provide cost-effective, high-performance, and reliable services, including VPS, Web, Dedicated, and Colocation. | | [**indifferent broccoli**](https://indifferentbroccoli.com/) | indifferent broccoli is a game server hosting and rental company. With them, you get top-notch computer power for your gaming sessions. They destroy lag, latency, and complexity--letting you focus on the fun stuff. | +| [**Infraly, LLC**](https://infraly.co/) | Infraly is an infrastructure company powering the next generation of online services. Through their brands, Infraly delivers cutting-edge solutions across multiple markets. Their vertically integrated approach provides unmatched performance, scalability, and reliability, giving our customers full control. | +| [**MineStrator**](https://minestrator.com/) | MineStrator is a game server hosting provider. Looking for the most high-end French hosting company for your Minecraft server? More than 24,000 members on our Discord trust us. Give us a try! | +| [**Physgun**](https://physgun.com/) | Physgun is a game server hosting provider. Most providers rent rack space and rebrand a panel. At Physgun, they engineer the performance, write the features, and staff the support. Physgun truly is game hosting perfected! | +| [**WISP**](https://wisp.gg/) | WISP is an industry-leading SaaS platform for game server management, designed for hosting companies, gaming organizations, and enthusiasts. WISP combines modern, intuitive interfaces with powerful tools, making server deployment and administration seamless, scalable, and efficient. ## Documentation -* [Panel Documentation](https://pterodactyl.io/panel/1.0/getting_started.html) -* [Wings Documentation](https://pterodactyl.io/wings/1.0/installing.html) -* [Community Guides](https://pterodactyl.io/community/about.html) +* [Panel Documentation](https://docs.pterodactyl.io/v1/panel/getting-started) +* [Wings Documentation](https://docs.pterodactyl.io/v1/wings/) +* [Community Guides](https://docs.pterodactyl.io/v1/guides/panel-installation/debian) * Or, get additional help [via Discord](https://discord.gg/pterodactyl) ## Reporting Issues diff --git a/WEBSOCKET_EVENTS.md b/WEBSOCKET_EVENTS.md index d77f4465c..c247a167a 100644 --- a/WEBSOCKET_EVENTS.md +++ b/WEBSOCKET_EVENTS.md @@ -100,6 +100,7 @@ Real-time backup creation/restoration progress. { "event": "backup progress", "args": [{ + "backup_id": "47363ce7-d70a-430e-8e75-6dc87c8d016d", "type": "create", "percentage": 45, "bytes_written": 471859200, @@ -109,10 +110,35 @@ Real-time backup creation/restoration progress. ``` **Fields:** -- `type`: `"create"` or `"restore"` +- `backup_id`: UUID of the backup +- `type`: `"create"`, `"restore"`, or `"download"` (S3 backup being downloaded for a restore) - `percentage`: 0-100, or -1 for error -- `bytes_written`: Bytes processed -- `bytes_total`: Total bytes (create only) +- `bytes_written`: Bytes processed (omitted when 0) +- `bytes_total`: Total bytes (omitted when unknown) + +Requires the `backup.read` permission (same gate as `backup completed` / `backup restore completed`). + +#### `download progress` ⭐ *New* +Progress of a remote file pull (`POST /api/servers/:server/files/pull` with `foreground: false`). +```json +{ + "event": "download progress", + "args": [{ + "identifier": "download-uuid", + "filename": "world.zip", + "directory": "/", + "url": "https://example.com/world.zip", + "percentage": 45, + "bytes_written": 471859200, + "bytes_total": 1048576000, + "status": "downloading" + }] +} +``` + +- `status`: `"downloading"`, `"completed"` (percentage 100) or `"failed"` (percentage -1); a final event is always sent +- Throttled to 250 ms; `bytes_total` is 0 when the size is unknown +- Requires the `file.read` permission #### `backup completed` Backup creation finished. @@ -122,8 +148,8 @@ Backup creation finished. "args": [{ "uuid": "backup-uuid", "is_successful": true, - "checksum": "sha1-hash", - "checksum_type": "sha1", + "checksum": "sha256-hash", + "checksum_type": "sha256", "file_size": 1048576000 }] } @@ -141,12 +167,12 @@ Backup restoration finished. ### 📁 **Activity Events** ⭐ *Enhanced* #### `activity` -File operations and server activities. +File operations and server activities, pushed live in addition to being stored for the Panel. Requires the `activity.read` permission. ```json { "event": "activity", "args": [{ - "event": "server:file.write", + "event": "server:sftp.write", "user": "user-uuid", "metadata": { "file": "config/server.properties" @@ -156,15 +182,16 @@ File operations and server activities. ``` **Common Activity Types:** -- `server:file.write` - File created/modified -- `server:file.delete` - File deleted -- `server:file.rename` - File renamed -- `server:file.create-directory` - Directory created -- `server:file.compress` - Files compressed -- `server:file.decompress` - Archive extracted +- `server:sftp.create` / `server:sftp.write` - File created / modified (SFTP or the file API) +- `server:sftp.delete` - File deleted +- `server:sftp.rename` - File renamed +- `server:sftp.create-directory` - Directory created +- `server:file.downloaded` - Remote file pulled +- `server:file.compressed` / `server:file.decompressed` - Archive created / extracted +- `server:file.chmod` - Permissions changed - `server:console.command` - Console command executed -- `server:power.start` - Server started -- `server:power.stop` - Server stopped +- `server:power.start` / `server:power.stop` - Power actions +- `backup_failed` - All backup retries failed (different payload: `{event, timestamp, backup_id, server_id, error, total_attempts, message}`) ### 🔄 **Transfer Events** @@ -204,7 +231,8 @@ Server has been deleted from the system. | `status` | On state change | None | | `stats` | Every 1-2 seconds | Built-in | | `console output` | Real-time | None | -| `backup progress` | Max 2/second | 500ms throttling | +| `backup progress` | Max 4/second | 250 ms throttling (`download` type unthrottled) | +| `download progress` | Max 4/second | 250 ms throttling | | `activity` | On action | None | | `install output` | Real-time | None | | `daemon message` | As needed | None | diff --git a/cmd/configure.go b/cmd/configure.go index cc272ecb5..097f63dab 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -3,6 +3,7 @@ package cmd import ( "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -48,12 +49,20 @@ func init() { func configureCmdRun(cmd *cobra.Command, args []string) { if configureArgs.AllowInsecure { http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: true, //nolint:gosec // opt-in via --allow-insecure } } - if _, err := os.Stat(configureArgs.ConfigPath); err == nil && !configureArgs.Override { - survey.AskOne(&survey.Confirm{Message: "Override existing configuration file"}, &configureArgs.Override) + // The configuration is written to --config-path, or to the global --config + // path if only that was given. + target := configureArgs.ConfigPath + if cmd != nil && !cmd.Flags().Changed("config-path") && cmd.Flags().Changed("config") { + target = configPath + } + + if _, err := os.Stat(target); err == nil && !configureArgs.Override { + // A failed or interrupted prompt leaves Override false, which aborts below. + _ = survey.AskOne(&survey.Confirm{Message: "Override existing configuration file"}, &configureArgs.Override) if !configureArgs.Override { fmt.Println("Aborting process; a configuration file already exists for this node.") os.Exit(1) @@ -124,7 +133,6 @@ func configureCmdRun(cmd *cobra.Command, args []string) { panic(err) } - fmt.Printf("%+v", req.Header) fmt.Println(req.URL.String()) res, err := c.Do(req) @@ -144,25 +152,50 @@ func configureCmdRun(cmd *cobra.Command, args []string) { os.Exit(1) } - b, err := io.ReadAll(res.Body) - - cfg, err := config.NewAtPath(configPath) + b, err := io.ReadAll(io.LimitReader(res.Body, maxConfigurationResponseSize)) if err != nil { panic(err) } - if err := json.Unmarshal(b, cfg); err != nil { - panic(err) + if err := writePanelConfiguration(target, configureArgs.PanelURL, b); err != nil { + fmt.Println("Failed to configure wings.\n", err.Error()) + os.Exit(1) } + fmt.Println("Successfully configured wings.") +} + +// maxConfigurationResponseSize is the largest node configuration that is read +// from the Panel. +const maxConfigurationResponseSize = 1 << 20 + +// writePanelConfiguration applies the node configuration returned by the Panel +// to the configuration file at path, keeping any other settings already in it. +func writePanelConfiguration(path string, panelURL string, body []byte) error { + cfg, err := config.Load(path) + if errors.Is(err, os.ErrNotExist) { + cfg, err = config.NewAtPath(path) + } + if err != nil { + return err + } + + // Only take the values from the Panel that it is allowed to set. + p := cfg.PanelConfiguration() + if err := json.Unmarshal(body, &p); err != nil { + return err + } + cfg.ApplyPanelConfiguration(p) + // Manually specify the Panel URL as it won't be decoded from JSON. - cfg.PanelLocation = configureArgs.PanelURL + cfg.PanelLocation = panelURL - if err = config.WriteToDisk(cfg); err != nil { - panic(err) + // The Panel returns the token itself. + if err := cfg.ResolveToken(true); err != nil { + return err } - fmt.Println("Successfully configured wings.") + return config.WriteToDisk(cfg) } func getRequest() (*http.Request, error) { diff --git a/cmd/configure_test.go b/cmd/configure_test.go new file mode 100644 index 000000000..5aade0b3e --- /dev/null +++ b/cmd/configure_test.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" +) + +// useConfigureArgs points the configure command at a fake Panel and the given +// --config-path and --config files, restoring the package state afterwards. +func useConfigureArgs(t *testing.T, panelURL, configurePath, globalPath string) { + t.Helper() + prevArgs := configureArgs + prevConfigPath := configPath + t.Cleanup(func() { + configureArgs = prevArgs + configPath = prevConfigPath + }) + + configureArgs.PanelURL = panelURL + configureArgs.Token = "test-token" + configureArgs.Node = "1" + configureArgs.ConfigPath = configurePath + configureArgs.Override = false + configureArgs.AllowInsecure = false + configPath = globalPath +} + +// newFakePanel serves the node configuration endpoint with the given body. +func newFakePanel(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/application/nodes/1/configuration" || r.Header.Get("Authorization") != "Bearer test-token" { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func clearTokenEnvironment(t *testing.T) { + t.Helper() + t.Setenv("WINGS_TOKEN_ID", "") + t.Setenv("WINGS_TOKEN", "") +} + +func TestConfigureWritesToConfigPath(t *testing.T) { + clearTokenEnvironment(t) + dir := t.TempDir() + configurePath := filepath.Join(dir, "configure.yml") + globalPath := filepath.Join(dir, "global.yml") + const existing = "uuid: existing-node\n" + require.NoError(t, os.WriteFile(globalPath, []byte(existing), 0o600)) + + panel := newFakePanel(t, `{"uuid":"11111111-2222-3333-4444-555555555555","token_id":"panel-id","token":"panel-token"}`) + useConfigureArgs(t, panel.URL, configurePath, globalPath) + + configureCmdRun(nil, nil) + + got, err := os.ReadFile(globalPath) + require.NoError(t, err) + require.Equal(t, existing, string(got), "the --config file should not be written when --config-path is given") + + cfg, err := config.Load(configurePath) + require.NoError(t, err) + require.Equal(t, "panel-token", cfg.AuthenticationToken) + require.Equal(t, panel.URL, cfg.PanelLocation) +} + +// The token from the Panel is used as it is. +func TestConfigureRequiresTokenValue(t *testing.T) { + clearTokenEnvironment(t) + tokenFile := filepath.Join(t.TempDir(), "token") + require.NoError(t, os.WriteFile(tokenFile, []byte("file-token"), 0o600)) + + for _, token := range []string{"file://" + tokenFile, "$HOME"} { + path := filepath.Join(t.TempDir(), "config.yml") + body := `{"token_id":"panel-id","token":"` + token + `"}` + + require.Error(t, writePanelConfiguration(path, "http://panel.test", []byte(body)), token) + require.NoFileExists(t, path) + } +} + +// Configuring an existing node keeps the settings the Panel does not manage. +func TestConfigureKeepsExistingSettings(t *testing.T) { + clearTokenEnvironment(t) + path := filepath.Join(t.TempDir(), "config.yml") + require.NoError(t, os.WriteFile(path, []byte(`token_id: old-id +token: old-token +allowed_mounts: + - /srv/shared +docker: + registries: + registry.example.com: + username: user + password: pass +`), 0o600)) + + require.NoError(t, writePanelConfiguration(path, "http://panel.test", []byte(`{"token_id":"new-id","token":"new-token"}`))) + + cfg, err := config.Load(path) + require.NoError(t, err) + require.Equal(t, "new-id", cfg.AuthenticationTokenId) + require.Equal(t, "new-token", cfg.AuthenticationToken) + require.Equal(t, []string{"/srv/shared"}, cfg.AllowedMounts) + require.Contains(t, cfg.Docker.Registries, "registry.example.com") +} diff --git a/cmd/diagnostics.go b/cmd/diagnostics.go index 7f2e734e1..4c9512aa4 100644 --- a/cmd/diagnostics.go +++ b/cmd/diagnostics.go @@ -34,6 +34,9 @@ const ( DefaultLogLines = 200 ) +// hastebinTimeout is how long uploading the report may take. +var hastebinTimeout = 30 * time.Second + var diagnosticsArgs struct { IncludeEndpoints bool IncludeLogs bool @@ -80,7 +83,7 @@ func diagnosticsCmdRun(*cobra.Command, []string) { Name: "ReviewBeforeUpload", Prompt: &survey.Confirm{ Message: "Do you want to review the collected data before uploading to " + diagnosticsArgs.HastebinURL + "?", - Help: "The data, especially the logs, might contain sensitive information, so you should review it. You will be asked again if you want to upload.", + Help: "The data, especially the logs, might contain sensitive information, so you should review it. You will be asked before anything is uploaded.", Default: true, }, }, @@ -109,28 +112,7 @@ func diagnosticsCmdRun(*cobra.Command, []string) { } printHeader(output, "Wings Configuration") - if err := config.FromFile(config.DefaultLocation); err != nil { - } - cfg := config.Get() - fmt.Fprintln(output, " Panel Location:", redact(cfg.PanelLocation)) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Internal Webserver:", redact(cfg.Api.Host), ":", cfg.Api.Port) - fmt.Fprintln(output, " SSL Enabled:", cfg.Api.Ssl.Enabled) - fmt.Fprintln(output, " SSL Certificate:", redact(cfg.Api.Ssl.CertificateFile)) - fmt.Fprintln(output, " SSL Key:", redact(cfg.Api.Ssl.KeyFile)) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " SFTP Server:", redact(cfg.System.Sftp.Address), ":", cfg.System.Sftp.Port) - fmt.Fprintln(output, " SFTP Read-Only:", cfg.System.Sftp.ReadOnly) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Root Directory:", cfg.System.RootDirectory) - fmt.Fprintln(output, " Logs Directory:", cfg.System.LogDirectory) - fmt.Fprintln(output, " Data Directory:", cfg.System.Data) - fmt.Fprintln(output, " Archive Directory:", cfg.System.ArchiveDirectory) - fmt.Fprintln(output, " Backup Directory:", cfg.System.BackupDirectory) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Username:", cfg.System.Username) - fmt.Fprintln(output, " Server Time:", time.Now().Format(time.RFC1123Z)) - fmt.Fprintln(output, " Debug Mode:", cfg.Debug) + cfg := writeConfiguration(output) printHeader(output, "Docker: Info") if dockerErr == nil { @@ -171,7 +153,7 @@ func diagnosticsCmdRun(*cobra.Command, []string) { if cfg != nil { p = path.Join(cfg.System.LogDirectory, "wings.log") } - if c, err := exec.Command("tail", "-n", strconv.Itoa(diagnosticsArgs.LogLines), p).Output(); err != nil { + if c, err := exec.Command("tail", "-n", strconv.Itoa(diagnosticsArgs.LogLines), p).Output(); err != nil { //nolint:gosec // fixed binary, no shell; path comes from the operator's config fmt.Fprintln(output, "No logs found or an error occurred.") } else { fmt.Fprintf(output, "%s\n", string(c)) @@ -180,25 +162,28 @@ func diagnosticsCmdRun(*cobra.Command, []string) { fmt.Fprintln(output, "Logs redacted.") } - if !diagnosticsArgs.IncludeEndpoints { + if !diagnosticsArgs.IncludeEndpoints && cfg != nil { s := output.String() output.Reset() - s = strings.ReplaceAll(s, cfg.PanelLocation, "{redacted}") - s = strings.ReplaceAll(s, cfg.Api.Host, "{redacted}") - s = strings.ReplaceAll(s, cfg.Api.Ssl.CertificateFile, "{redacted}") - s = strings.ReplaceAll(s, cfg.Api.Ssl.KeyFile, "{redacted}") - s = strings.ReplaceAll(s, cfg.System.Sftp.Address, "{redacted}") + for _, v := range []string{cfg.PanelLocation, cfg.Api.Host, cfg.Api.Ssl.CertificateFile, cfg.Api.Ssl.KeyFile, cfg.System.Sftp.Address} { + // Replacing an empty string would insert the placeholder between every character. + if v != "" { + s = strings.ReplaceAll(s, v, "{redacted}") + } + } output.WriteString(s) } - fmt.Println("\n--------------- generated report ---------------") - fmt.Println(output.String()) - fmt.Print("--------------- end of report ---------------\n\n") - - upload := !diagnosticsArgs.ReviewBeforeUpload - if !upload { - survey.AskOne(&survey.Confirm{Message: "Upload to " + diagnosticsArgs.HastebinURL + "?", Default: false}, &upload) + if diagnosticsArgs.ReviewBeforeUpload { + fmt.Println("\n--------------- generated report ---------------") + fmt.Println(output.String()) + fmt.Print("--------------- end of report ---------------\n\n") } + + // Always ask before uploading, since the report can contain sensitive + // information. A failed or interrupted prompt leaves upload false. + var upload bool + _ = survey.AskOne(&survey.Confirm{Message: "Upload to " + diagnosticsArgs.HastebinURL + "?", Default: false}, &upload) if upload { u, err := uploadToHastebin(diagnosticsArgs.HastebinURL, output.String()) if err == nil { @@ -207,6 +192,37 @@ func diagnosticsCmdRun(*cobra.Command, []string) { } } +// writeConfiguration writes the relevant parts of the configuration file Wings +// was started with, and returns it if it could be read. +func writeConfiguration(w io.Writer) *config.Configuration { + var cfg *config.Configuration + if err := config.FromFile(configPath); err != nil { + fmt.Fprintln(w, "Failed to load configuration:", err) + } else { + cfg = config.Get() + fmt.Fprintln(w, " Panel Location:", redact(cfg.PanelLocation)) + fmt.Fprintln(w, "") + fmt.Fprintln(w, " Internal Webserver:", redact(cfg.Api.Host), ":", cfg.Api.Port) + fmt.Fprintln(w, " SSL Enabled:", cfg.Api.Ssl.Enabled) + fmt.Fprintln(w, " SSL Certificate:", redact(cfg.Api.Ssl.CertificateFile)) + fmt.Fprintln(w, " SSL Key:", redact(cfg.Api.Ssl.KeyFile)) + fmt.Fprintln(w, "") + fmt.Fprintln(w, " SFTP Server:", redact(cfg.System.Sftp.Address), ":", cfg.System.Sftp.Port) + fmt.Fprintln(w, " SFTP Read-Only:", cfg.System.Sftp.ReadOnly) + fmt.Fprintln(w, "") + fmt.Fprintln(w, " Root Directory:", cfg.System.RootDirectory) + fmt.Fprintln(w, " Logs Directory:", cfg.System.LogDirectory) + fmt.Fprintln(w, " Data Directory:", cfg.System.Data) + fmt.Fprintln(w, " Archive Directory:", cfg.System.ArchiveDirectory) + fmt.Fprintln(w, " Backup Directory:", cfg.System.BackupDirectory) + fmt.Fprintln(w, "") + fmt.Fprintln(w, " Username:", cfg.System.Username) + fmt.Fprintln(w, " Server Time:", time.Now().Format(time.RFC1123Z)) + fmt.Fprintln(w, " Debug Mode:", cfg.Debug) + } + return cfg +} + func getDockerInfo() (types.Version, dockersystem.Info, error) { client, err := environment.Docker() if err != nil { @@ -230,18 +246,27 @@ func uploadToHastebin(hbUrl, content string) (string, error) { return "", err } u.Path = path.Join(u.Path, "documents") - res, err := http.Post(u.String(), "text/plain", r) - if err != nil || res.StatusCode < 200 || res.StatusCode >= 300 { + client := &http.Client{Timeout: hastebinTimeout} + res, err := client.Post(u.String(), "text/plain", r) + if err != nil { fmt.Println("Failed to upload report to ", u.String(), err) return "", err } + defer res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + fmt.Println("Failed to upload report to ", u.String(), res.Status) + return "", errors.New("unexpected response status: " + res.Status) + } pres := make(map[string]interface{}) - body, err := io.ReadAll(res.Body) + body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) if err != nil { fmt.Println("Failed to parse response.", err) return "", err } - json.Unmarshal(body, &pres) + if err := json.Unmarshal(body, &pres); err != nil { + fmt.Println("Failed to parse response.", err) + return "", err + } if key, ok := pres["key"].(string); ok { u, _ := url.Parse(hbUrl) u.Path = path.Join(u.Path, key) diff --git a/cmd/diagnostics_test.go b/cmd/diagnostics_test.go new file mode 100644 index 000000000..d2d0fc446 --- /dev/null +++ b/cmd/diagnostics_test.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" +) + +// The report describes the configuration file Wings was started with. +func TestDiagnosticsReadsConfigFromConfigPath(t *testing.T) { + prevConfigPath := configPath + prevArgs := diagnosticsArgs + t.Cleanup(func() { + configPath = prevConfigPath + diagnosticsArgs = prevArgs + }) + + p := filepath.Join(t.TempDir(), "config.yml") + cfg, err := config.NewAtPath(p) + require.NoError(t, err) + cfg.AuthenticationTokenId = "token-id" + cfg.AuthenticationToken = "test-token" + cfg.PanelLocation = "https://panel.example.test" + require.NoError(t, config.WriteToDisk(cfg)) + + configPath = p + diagnosticsArgs.IncludeEndpoints = true + + var out strings.Builder + got := writeConfiguration(&out) + require.NotNil(t, got, out.String()) + require.Contains(t, out.String(), "https://panel.example.test") +} + +// An upload that gets no response times out. +func TestDiagnosticsUploadTimesOut(t *testing.T) { + prev := hastebinTimeout + hastebinTimeout = 200 * time.Millisecond + t.Cleanup(func() { hastebinTimeout = prev }) + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { close(release) }) + + done := make(chan error, 1) + go func() { + _, err := uploadToHastebin(srv.URL, "report") + done <- err + }() + + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("expected the upload to time out") + } +} + +func TestDiagnosticsUploadReturnsDocumentURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/documents" { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write([]byte(`{"key":"abc123"}`)) + })) + t.Cleanup(srv.Close) + + u, err := uploadToHastebin(srv.URL, "report") + require.NoError(t, err) + require.Equal(t, srv.URL+"/abc123", u) +} diff --git a/cmd/root.go b/cmd/root.go index 0310fcf7f..608aa3c83 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" log2 "log" + "net" "net/http" - _ "net/http/pprof" + _ "net/http/pprof" //nolint:gosec // only served on localhost with --pprof; nothing else uses http.DefaultServeMux "os" + "os/signal" "path" "path/filepath" "runtime" @@ -19,7 +21,7 @@ import ( "github.com/NYTimes/logrotate" "github.com/apex/log" "github.com/apex/log/handlers/multi" - "github.com/docker/docker/client" + cerrdefs "github.com/containerd/errdefs" "github.com/gammazero/workerpool" "github.com/mitchellh/colorstring" "github.com/spf13/cobra" @@ -30,6 +32,7 @@ import ( "github.com/Rene-Roscher/wings/environment" "github.com/Rene-Roscher/wings/internal/cron" "github.com/Rene-Roscher/wings/internal/database" + "github.com/Rene-Roscher/wings/internal/network" "github.com/Rene-Roscher/wings/loggers/cli" "github.com/Rene-Roscher/wings/remote" "github.com/Rene-Roscher/wings/router" @@ -98,7 +101,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { if ok, _ := cmd.Flags().GetBool("ignore-certificate-errors"); ok { log.Warn("running with --ignore-certificate-errors: TLS certificate host chains and name will not be verified") http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, + InsecureSkipVerify: true, //nolint:gosec // opt-in via --ignore-certificate-errors } } @@ -225,7 +228,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // point. If we didn't do this, and you pruned all the images and then started wings you could // end up waiting a long period of time for all the images to be re-pulled on Wings boot rather // than when the server itself is started. - if err != nil && !client.IsErrNotFound(err) { + if err != nil && !cerrdefs.IsNotFound(err) { s.Log().WithField("error", err).Error("error checking server environment status") } @@ -285,9 +288,14 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { s.StartAsync() } + httpConnections, sftpConnections := connectionLimits() + log.WithFields(log.Fields{"http": httpConnections, "sftp": sftpConnections}).Info("limiting the number of open connections, raise the open file limit for Wings to allow more") + go func() { // Run the SFTP server. - if err := sftp.New(manager).Run(); err != nil { + srv := sftp.New(manager) + srv.MaxConnections = sftpConnections + if err := srv.Run(); err != nil { log.WithError(err).Fatal("failed to initialize the sftp server") return } @@ -304,7 +312,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { sys := config.Get().System // Ensure the archive directory exists. - if err := os.MkdirAll(sys.ArchiveDirectory, 0o755); err != nil { + if err := os.MkdirAll(sys.ArchiveDirectory, 0o700); err != nil { log.WithField("error", err).Error("failed to create archive directory") } @@ -315,7 +323,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { }() // Ensure the backup directory exists. - if err := os.MkdirAll(sys.BackupDirectory, 0o755); err != nil { + if err := os.MkdirAll(sys.BackupDirectory, 0o700); err != nil { log.WithField("error", err).Error("failed to create backup directory") } @@ -335,11 +343,19 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Create a new HTTP server instance to handle inbound requests from the Panel // and external clients. - s := &http.Server{ - Addr: api.Host + ":" + strconv.Itoa(api.Port), - Handler: router.Configure(manager, pclient), - TLSConfig: config.DefaultTLSConfig, + s := router.NewServer(api.Host+":"+strconv.Itoa(api.Port), router.Configure(manager, pclient), config.DefaultTLSConfig) + listener, err := net.Listen("tcp", s.Addr) + if err != nil { + log.WithField("error", err).Fatal("failed to configure HTTP server") } + listener = network.LimitListener(listener, httpConnections, "http") + + // Save the state of every server and stop serving requests when Wings is asked + // to stop. The servers themselves keep running, and are picked up again when + // Wings next starts. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + go shutdownOnSignal(signals, manager, s) profile, _ := cmd.Flags().GetBool("pprof") if profile { @@ -351,7 +367,11 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { profilePort, _ := cmd.Flags().GetInt("pprof-port") go func() { - http.ListenAndServe(fmt.Sprintf("localhost:%d", profilePort), nil) + srv := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", profilePort), + ReadHeaderTimeout: time.Second * 10, + } + _ = srv.ListenAndServe() }() } @@ -371,12 +391,12 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Start the autocert server. go func() { - if err := http.ListenAndServe(":http", m.HTTPHandler(nil)); err != nil { + if err := router.NewServer(":http", m.HTTPHandler(nil), nil).ListenAndServe(); err != nil { log.WithError(err).Error("failed to serve autocert http server") } }() // Start the main http server with TLS using autocert. - if err := s.ListenAndServeTLS("", ""); err != nil { + if err := s.ServeTLS(listener, "", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { log.WithFields(log.Fields{"auto_tls": true, "tls_hostname": tlshostname, "error": err}).Fatal("failed to configure HTTP server using auto-tls") } return @@ -385,17 +405,53 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Check if main http server should run with TLS. Otherwise, reset the TLS // config on the server and then serve it over normal HTTP. if api.Ssl.Enabled { - if err := s.ListenAndServeTLS(api.Ssl.CertificateFile, api.Ssl.KeyFile); err != nil { + if err := s.ServeTLS(listener, api.Ssl.CertificateFile, api.Ssl.KeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) { log.WithFields(log.Fields{"auto_tls": false, "error": err}).Fatal("failed to configure HTTPS server") } return } s.TLSConfig = nil - if err := s.ListenAndServe(); err != nil { + if err := s.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { log.WithField("error", err).Fatal("failed to configure HTTP server") } } +// shutdownOnSignal waits for a signal, then saves the state of every server and +// stops the HTTP server, which returns from Serve so that Wings can exit. +func shutdownOnSignal(signals <-chan os.Signal, manager *server.Manager, srv *http.Server) { + sig := <-signals + log.WithField("signal", sig.String()).Info("received signal, shutting down") + if err := manager.PersistStates(); err != nil { + log.WithField("error", err).Warn("failed to persist server states to disk") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + log.WithField("error", err).Warn("failed to stop the HTTP server cleanly") + _ = srv.Close() + } +} + +// connectionLimits returns the number of connections the HTTP and SFTP servers +// may each have open at once. Every connection uses a file descriptor, so the +// limits are based on the open file limit for the process, leaving room for +// everything else Wings does. Raise the open file limit to allow more. +func connectionLimits() (int, int) { + var rl syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rl); err != nil { + return 3072, 1024 + } + limit := rl.Cur + if limit > 1<<20 { + limit = 1 << 20 + } + available := int(limit) - 1024 + if available < int(limit)/2 { + available = int(limit) / 2 + } + return available * 3 / 4, available / 4 +} + // Reads the configuration from the disk and then sets up the global singleton // with all the configuration values. func initConfig() { @@ -452,8 +508,8 @@ __ [blue][bold]Pterodactyl[reset] _____/___/_______ _______ ______ Copyright © 2018 - %d Dane Everitt & Contributors Website: https://pterodactyl.io - Source: https://github.com/Rene-Roscher/wings -License: https://github.com/Rene-Roscher/wings/blob/develop/LICENSE + Source: https://github.com/emeraldhost/wings +License: https://github.com/emeraldhost/wings/blob/develop/LICENSE This software is made available under the terms of the MIT license. The above copyright notice and this permission notice shall be included diff --git a/cmd/shutdown_test.go b/cmd/shutdown_test.go new file mode 100644 index 000000000..e3240a278 --- /dev/null +++ b/cmd/shutdown_test.go @@ -0,0 +1,45 @@ +package cmd + +import ( + "net" + "net/http" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/server" +) + +// On a signal the server states are saved and the HTTP server stops, so that +// Wings exits instead of being killed. +func TestShutdownOnSignalSavesStatesAndStopsServer(t *testing.T) { + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.System.RootDirectory = t.TempDir() + config.Set(cfg) + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := &http.Server{ReadHeaderTimeout: time.Second} + served := make(chan error, 1) + go func() { served <- srv.Serve(l) }() + + signals := make(chan os.Signal, 1) + go shutdownOnSignal(signals, server.NewEmptyManager(nil), srv) + signals <- syscall.SIGTERM + + select { + case err := <-served: + require.ErrorIs(t, err, http.ErrServerClosed) + case <-time.After(5 * time.Second): + t.Fatal("expected the HTTP server to stop") + } + _, err = os.Stat(config.Get().System.GetStatesPath()) + require.NoError(t, err, "expected the server states to be saved") +} diff --git a/config/config.go b/config/config.go index 4c8b13a40..359520152 100644 --- a/config/config.go +++ b/config/config.go @@ -69,6 +69,14 @@ type SftpConfiguration struct { Port int `default:"2022" json:"bind_port" yaml:"bind_port"` // If set to true, no write actions will be allowed on the SFTP server. ReadOnly bool `default:"false" yaml:"read_only"` + // The maximum number of connections a single user may have open at once, or + // zero to use the default of 32. + MaxConnectionsPerUser int `json:"-" yaml:"max_connections_per_user"` + // The maximum number of connections from a single IP address, or IPv6 /64 + // network, that may be authenticating at once, or zero to use the default of + // 16. Raise this if clients connect through a proxy, since they then all + // appear to come from the address of the proxy. + MaxAuthenticatingConnectionsPerIP int `json:"-" yaml:"max_authenticating_connections_per_ip"` } // ApiConfiguration defines the configuration for the internal API that is @@ -92,6 +100,11 @@ type ApiConfiguration struct { // servers. DisableRemoteDownload bool `json:"-" yaml:"disable_remote_download"` + // RemoteDownloadAllowlist allows remote downloads of files into server directories to + // connect to otherwise blocked private/internal destinations. Entries may be hostnames, + // IP addresses, or CIDR ranges. + RemoteDownloadAllowlist []string `json:"-" yaml:"remote_download_allowlist"` + // The maximum size for files uploaded through the Panel in MB. UploadLimit int64 `default:"100" json:"upload_limit" yaml:"upload_limit"` @@ -369,7 +382,7 @@ type Configuration struct { // additional origins. AllowedOrigins []string `json:"allowed_origins" yaml:"allowed_origins"` - // AllowCORSPrivateNetwork sets the `Access-Control-Request-Private-Network` header which + // AllowCORSPrivateNetwork sets the `Access-Control-Allow-Private-Network` header which // allows client browsers to make requests to internal IP addresses over HTTP. This setting // is only required by users running Wings without SSL certificates and using internal IP // addresses in order to connect. Most users should NOT enable this setting. @@ -379,6 +392,65 @@ type Configuration struct { IgnorePanelConfigUpdates bool `json:"ignore_panel_config_updates" yaml:"ignore_panel_config_updates"` } +// PanelConfiguration contains the only configuration values the Panel is allowed +// to set on a node. Everything else in the configuration controls how Wings itself +// runs on the host and must only be changed by the node operator. +type PanelConfiguration struct { + Debug bool `json:"debug"` + Uuid string `json:"uuid"` + TokenId string `json:"token_id"` + Token string `json:"token"` + Api struct { + Host string `json:"host"` + Port int `json:"port"` + Ssl struct { + Enabled bool `json:"enabled"` + CertificateFile string `json:"cert"` + KeyFile string `json:"key"` + } `json:"ssl"` + UploadLimit int64 `json:"upload_limit"` + } `json:"api"` + System struct { + Sftp struct { + Port int `json:"bind_port"` + } `json:"sftp"` + } `json:"system"` +} + +// PanelConfiguration returns the values in the configuration that the Panel is +// allowed to set. +func (c *Configuration) PanelConfiguration() PanelConfiguration { + var p PanelConfiguration + p.Debug = c.Debug + p.Uuid = c.Uuid + p.TokenId = c.AuthenticationTokenId + p.Token = c.AuthenticationToken + p.Api.Host = c.Api.Host + p.Api.Port = c.Api.Port + p.Api.Ssl.Enabled = c.Api.Ssl.Enabled + p.Api.Ssl.CertificateFile = c.Api.Ssl.CertificateFile + p.Api.Ssl.KeyFile = c.Api.Ssl.KeyFile + p.Api.UploadLimit = c.Api.UploadLimit + p.System.Sftp.Port = c.System.Sftp.Port + return p +} + +// ApplyPanelConfiguration sets the values in the configuration that the Panel is +// allowed to set. +func (c *Configuration) ApplyPanelConfiguration(p PanelConfiguration) { + c.Debug = p.Debug + c.Uuid = p.Uuid + c.AuthenticationTokenId = p.TokenId + c.AuthenticationToken = p.Token + c.Api.Host = p.Api.Host + c.Api.Port = p.Api.Port + c.Api.Ssl.Enabled = p.Api.Ssl.Enabled + c.Api.Ssl.CertificateFile = p.Api.Ssl.CertificateFile + c.Api.Ssl.KeyFile = p.Api.Ssl.KeyFile + c.Api.UploadLimit = p.Api.UploadLimit + c.System.Sftp.Port = p.System.Sftp.Port +} + // NewAtPath creates a new struct and set the path where it should be stored. // This function does not modify the currently stored global configuration. func NewAtPath(path string) (*Configuration, error) { @@ -574,13 +646,13 @@ func EnsurePterodactylUser() error { command = fmt.Sprintf("adduser -S -D -H -G %[1]s -s /sbin/nologin %[1]s", _config.System.Username) // We have to create the group first on Alpine, so do that here before continuing on // to the user creation process. - if _, err := exec.Command("addgroup", "-S", _config.System.Username).Output(); err != nil { + if _, err := exec.Command("addgroup", "-S", _config.System.Username).Output(); err != nil { //nolint:gosec // no shell; username comes from the operator's config return err } } split := strings.Split(command, " ") - if _, err := exec.Command(split[0], split[1:]...).Output(); err != nil { + if _, err := exec.Command(split[0], split[1:]...).Output(); err != nil { //nolint:gosec // no shell; username comes from the operator's config return err } u, err = user.Lookup(_config.System.Username) @@ -605,7 +677,7 @@ container:x:%d: nogroup:x:65534:`, _config.System.User.Gid, )) - if err := os.WriteFile(filepath.Join(passwd.Directory, "group"), v, 0o644); err != nil { + if err := os.WriteFile(filepath.Join(passwd.Directory, "group"), v, 0o644); err != nil { //nolint:gosec // mounted as /etc/group, which containers must be able to read return fmt.Errorf("failed to write file to %s/group: %v", passwd.Directory, err) } @@ -617,25 +689,34 @@ nobody:x:65534:65534::/var/empty:/bin/sh _config.System.User.Uid, _config.System.User.Gid, )) - if err := os.WriteFile(filepath.Join(passwd.Directory, "passwd"), v, 0o644); err != nil { + if err := os.WriteFile(filepath.Join(passwd.Directory, "passwd"), v, 0o644); err != nil { //nolint:gosec // mounted as /etc/passwd, which containers must be able to read return fmt.Errorf("failed to write file to %s/passwd: %v", passwd.Directory, err) } return nil } -// FromFile reads the configuration from the provided file and stores it in the -// global singleton for this instance. -func FromFile(path string) error { - b, err := os.ReadFile(path) +// Load reads the configuration from the provided file without resolving its +// token or storing it in the global singleton. +func Load(path string) (*Configuration, error) { + b, err := os.ReadFile(path) //nolint:gosec // the configuration file chosen by the operator if err != nil { - return err + return nil, err } c, err := NewAtPath(path) if err != nil { - return err + return nil, err } - if err := yaml.Unmarshal(b, c); err != nil { + return nil, err + } + return c, nil +} + +// FromFile reads the configuration from the provided file and stores it in the +// global singleton for this instance. +func FromFile(path string) error { + c, err := Load(path) + if err != nil { return err } @@ -698,14 +779,14 @@ func ConfigureDirectories() error { if _config.System.Passwd.Enable { log.WithField("path", _config.System.Passwd.Directory).Debug("ensuring passwd directory exists") - if err := os.MkdirAll(_config.System.Passwd.Directory, 0o755); err != nil { + if err := os.MkdirAll(_config.System.Passwd.Directory, 0o755); err != nil { //nolint:gosec // files in here are mounted into containers return err } } if _config.System.MachineID.Enable { log.WithField("path", _config.System.MachineID.Directory).Debug("ensuring machine-id directory exists") - if err := os.MkdirAll(_config.System.MachineID.Directory, 0o755); err != nil { + if err := os.MkdirAll(_config.System.MachineID.Directory, 0o755); err != nil { //nolint:gosec // files in here are mounted into containers return err } } @@ -881,7 +962,7 @@ func Expand(v string) (string, error) { if strings.HasPrefix(v, filePrefix) { p := v[len(filePrefix):] - b, err := os.ReadFile(p) + b, err := os.ReadFile(p) //nolint:gosec // file:// values come from the operator's config if err != nil { return "", err } diff --git a/config/config_docker.go b/config/config_docker.go index 5f48b0ee9..0332cf178 100644 --- a/config/config_docker.go +++ b/config/config_docker.go @@ -233,7 +233,7 @@ type RegistryConfiguration struct { // Base64 returns the authentication for a given registry as a base64 encoded // string value. func (c RegistryConfiguration) Base64() (string, error) { - b, err := json.Marshal(registry.AuthConfig{ + b, err := json.Marshal(registry.AuthConfig{ //nolint:gosec // the Docker API needs these credentials to pull images Username: c.Username, Password: c.Password, }) diff --git a/dist/wings_test b/dist/wings_test deleted file mode 100755 index cfd4ce496..000000000 Binary files a/dist/wings_test and /dev/null differ diff --git a/environment/allocations.go b/environment/allocations.go index 337ccc4c2..6e0f75b62 100644 --- a/environment/allocations.go +++ b/environment/allocations.go @@ -65,28 +65,28 @@ func (a *Allocations) Bindings() nat.PortMap { // any reference to 127.0.0.1 with the IP of the pterodactyl0 network interface which will allow the // server to operate on a local address while still being accessible by other containers. func (a *Allocations) DockerBindings() nat.PortMap { - iface := config.Get().Docker.Network.Interface + network := config.Get().Docker.Network out := a.Bindings() // Loop over all the bindings for this container, and convert any that reference 127.0.0.1 // to use the pterodactyl0 network interface IP, as that is the true local for what people are // trying to do when creating servers. for p, binds := range out { - for i, alloc := range binds { - if alloc.HostIP != "127.0.0.1" { - continue - } - - // If using ISPN just delete the local allocation from the server. - if config.Get().Docker.Network.ISPN { - out[p] = append(out[p][:i], out[p][i+1:]...) - } else { - out[p][i] = nat.PortBinding{ - HostIP: iface, + kept := binds[:0] + for _, alloc := range binds { + if alloc.HostIP == "127.0.0.1" { + // If using ISPN just delete the local allocation from the server. + if network.ISPN { + continue + } + alloc = nat.PortBinding{ + HostIP: network.Interface, HostPort: alloc.HostPort, } } + kept = append(kept, alloc) } + out[p] = kept } return out diff --git a/environment/allocations_test.go b/environment/allocations_test.go new file mode 100644 index 000000000..6d07af171 --- /dev/null +++ b/environment/allocations_test.go @@ -0,0 +1,46 @@ +package environment + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" +) + +// Loopback bindings are removed when ISPN is enabled, including a port that is +// listed more than once. +func TestDockerBindingsIspnHandlesRepeatedLoopbackPort(t *testing.T) { + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.Docker.Network.ISPN = true + config.Set(cfg) + + a := Allocations{Mappings: map[string][]int{"127.0.0.1": {25565, 25565}}} + require.NotPanics(t, func() { + out := a.DockerBindings() + assert.Empty(t, out["25565/tcp"]) + assert.Empty(t, out["25565/udp"]) + }) +} + +// Without ISPN, loopback bindings use the Docker network interface and other +// bindings are kept. +func TestDockerBindingsReplaceLoopbackAddress(t *testing.T) { + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.Docker.Network.Interface = "172.18.0.1" + config.Set(cfg) + + a := Allocations{Mappings: map[string][]int{"127.0.0.1": {25565}, "10.0.0.5": {25565}}} + out := a.DockerBindings() + var ips []string + for _, b := range out["25565/tcp"] { + ips = append(ips, b.HostIP) + } + assert.ElementsMatch(t, []string{"172.18.0.1", "10.0.0.5"}, ips) +} diff --git a/environment/docker.go b/environment/docker.go index 1fd130301..1546d2437 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -7,6 +7,7 @@ import ( "emperror.dev/errors" "github.com/apex/log" + cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" @@ -40,7 +41,7 @@ func ConfigureDocker(ctx context.Context) error { nw := config.Get().Docker.Network resource, err := cli.NetworkInspect(ctx, nw.Name, network.InspectOptions{}) if err != nil { - if !client.IsErrNotFound(err) { + if !cerrdefs.IsNotFound(err) { return err } diff --git a/environment/docker/api.go b/environment/docker/api.go index 6e850c5ae..ff3297650 100644 --- a/environment/docker/api.go +++ b/environment/docker/api.go @@ -10,7 +10,9 @@ import ( "sync" "emperror.dev/errors" + "github.com/containerd/errdefs/pkg/errhttp" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/client" "github.com/docker/docker/errdefs" @@ -25,7 +27,6 @@ var ( ) type cliSettings struct { - enabled bool proto string host string scheme string @@ -49,7 +50,7 @@ func configure(c *client.Client) { // a large number of requests to this endpoint are spawned by Wings, and the // standard "encoding/json" shows its performance woes badly even with single // containers running. -func (e *Environment) ContainerInspect(ctx context.Context) (types.ContainerJSON, error) { +func (e *Environment) ContainerInspect(ctx context.Context) (container.InspectResponse, error) { configure(e.client) // Support feature flagging of this functionality so that if something goes @@ -59,7 +60,7 @@ func (e *Environment) ContainerInspect(ctx context.Context) (types.ContainerJSON return e.client.ContainerInspect(ctx, e.Id) } - var st types.ContainerJSON + var st container.InspectResponse req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/containers/"+e.Id+"/json", nil) if err != nil { return st, errors.WithStack(err) @@ -78,7 +79,7 @@ func (e *Environment) ContainerInspect(ctx context.Context) (types.ContainerJSON return st, errdefs.Unknown(err) } _ = res.Body.Close() - return st, errdefs.FromStatusCode(err, res.StatusCode) + return st, statusError(err, res.StatusCode) } defer res.Body.Close() @@ -87,7 +88,7 @@ func (e *Environment) ContainerInspect(ctx context.Context) (types.ContainerJSON return st, errors.Wrap(err, "failed to read response body from Docker") } if err := parseErrorFromResponse(res, body); err != nil { - return st, errdefs.FromStatusCode(err, res.StatusCode) + return st, statusError(err, res.StatusCode) } if err := json.Unmarshal(body, &st); err != nil { return st, errors.WithStack(err) @@ -120,3 +121,31 @@ func parseErrorFromResponse(res *http.Response, body []byte) error { return errors.Wrap(errors.New(emsg), "Error response from daemon") } + +// httpStatusError keeps the original error message while matching the error +// class (not found, conflict, etc.) for the response status code. +type httpStatusError struct { + err error + errdef error +} + +func (e *httpStatusError) Error() string { + return e.err.Error() +} + +func (e *httpStatusError) Unwrap() error { + return e.err +} + +func (e *httpStatusError) Is(target error) bool { + return errors.Is(e.errdef, target) +} + +// statusError is the equivalent of the Docker client's own handling of error +// responses, so callers can use cerrdefs.IsNotFound and friends on the result. +func statusError(err error, statusCode int) error { + if err == nil { + return nil + } + return &httpStatusError{err: err, errdef: errhttp.ToNative(statusCode)} +} diff --git a/environment/docker/api_test.go b/environment/docker/api_test.go new file mode 100644 index 000000000..247cf5721 --- /dev/null +++ b/environment/docker/api_test.go @@ -0,0 +1,38 @@ +package docker + +import ( + "net/http" + "testing" + + "emperror.dev/errors" + cerrdefs "github.com/containerd/errdefs" +) + +func TestStatusError(t *testing.T) { + base := errors.New("Error response from daemon: no such container") + + err := errors.WithStack(statusError(base, http.StatusNotFound)) + if !cerrdefs.IsNotFound(err) { + t.Errorf("expected a 404 to be reported as not found") + } + if err.Error() != base.Error() { + t.Errorf("expected message %q, got %q", base.Error(), err.Error()) + } + if !errors.Is(err, base) { + t.Errorf("expected the original error to be preserved") + } + + for _, code := range []int{http.StatusBadRequest, http.StatusConflict, http.StatusInternalServerError} { + if cerrdefs.IsNotFound(statusError(base, code)) { + t.Errorf("expected status %d not to be reported as not found", code) + } + } + + if !cerrdefs.IsConflict(statusError(base, http.StatusConflict)) { + t.Errorf("expected a 409 to be reported as a conflict") + } + + if statusError(nil, http.StatusNotFound) != nil { + t.Errorf("expected a nil error to stay nil") + } +} diff --git a/environment/docker/cgroup_burst.go b/environment/docker/cgroup_burst.go index ae521eef8..19ce15462 100644 --- a/environment/docker/cgroup_burst.go +++ b/environment/docker/cgroup_burst.go @@ -82,7 +82,7 @@ func writeBurstFile(pid int, burst int64) error { if err != nil { return err } - return os.WriteFile(f, []byte(strconv.FormatInt(burst, 10)), 0o644) + return os.WriteFile(f, []byte(strconv.FormatInt(burst, 10)), 0o644) //nolint:gosec // an existing cgroupfs file, the mode is never applied } // logBurstFailure warns the first time a burst cannot be applied and stays at diff --git a/environment/docker/command_test.go b/environment/docker/command_test.go new file mode 100644 index 000000000..f24e20f8c --- /dev/null +++ b/environment/docker/command_test.go @@ -0,0 +1,73 @@ +package docker + +import ( + "net" + "testing" + "time" + + "github.com/docker/docker/api/types" + + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/system" +) + +// attachedEnvironment returns an environment attached to one end of a pipe, +// returning the other end, which receives the process input. +func attachedEnvironment(t *testing.T) (*Environment, net.Conn) { + t.Helper() + local, remote := net.Pipe() + t.Cleanup(func() { + _ = local.Close() + _ = remote.Close() + }) + e := &Environment{meta: &Metadata{}, st: system.NewAtomicString(environment.ProcessRunningState)} + e.SetStream(&types.HijackedResponse{Conn: local}) + return e, remote +} + +// Commands may not contain control characters, other than tabs. +func TestSendCommandRejectsControlCharacters(t *testing.T) { + e, input := attachedEnvironment(t) + received := make(chan string, 1) + go func() { + b := make([]byte, 64) + n, _ := input.Read(b) + received <- string(b[:n]) + }() + + for _, c := range []string{"\x10\x11", "say \x10", "\x03", "\x1b[A", "\x7f"} { + if err := e.SendCommand(c); err == nil { + t.Errorf("expected command %q to be rejected", c) + } + } + if err := e.SendCommand("say\thello world"); err != nil { + t.Fatal(err) + } + select { + case got := <-received: + if got != "say\thello world\n" { + t.Fatalf("expected only the valid command to be sent, got %q", got) + } + case <-time.After(time.Second * 2): + t.Fatal("expected the valid command to be sent") + } +} + +// Sending a command times out if the process does not read its input. +func TestSendCommandDoesNotBlockForever(t *testing.T) { + previous := commandWriteTimeout + commandWriteTimeout = time.Millisecond * 200 + t.Cleanup(func() { commandWriteTimeout = previous }) + + e, _ := attachedEnvironment(t) + done := make(chan error, 1) + go func() { done <- e.SendCommand("say hello") }() + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error when the command could not be written") + } + case <-time.After(time.Second * 3): + t.Fatal("expected sending the command to time out") + } +} diff --git a/environment/docker/container.go b/environment/docker/container.go index 060ee5d09..ee88abc55 100644 --- a/environment/docker/container.go +++ b/environment/docker/container.go @@ -12,11 +12,11 @@ import ( "emperror.dev/errors" "github.com/apex/log" "github.com/buger/jsonparser" + cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" - "github.com/docker/docker/client" "github.com/Rene-Roscher/wings/config" "github.com/Rene-Roscher/wings/environment" @@ -25,6 +25,10 @@ import ( var ErrNotAttached = errors.Sentinel("not attached to instance") +// commandWriteTimeout is the amount of time allowed for a command to be written +// to the input of a server process. +var commandWriteTimeout = time.Second * 10 + // A custom console writer that allows us to keep a function blocked until the // given stream is properly closed. This does nothing special, only exists to // make a noop io.Writer. @@ -64,7 +68,7 @@ func (e *Environment) Attach(ctx context.Context) error { e.SetStream(&st) } - go func() { + go func() { //nolint:gosec // polling intentionally outlives ctx, see below // Don't use the context provided to the function, that'll cause the polling to // exit unexpectedly. We want a custom context for this, the one passed to the // function is to avoid a hang situation when trying to attach to a container. @@ -72,8 +76,13 @@ func (e *Environment) Attach(ctx context.Context) error { defer cancel() defer e.stream.Close() defer func() { - e.SetState(environment.ProcessOfflineState) e.SetStream(nil) + // The stream can end without the container stopping, in which case it is + // attached to again rather than treating the server as stopped. + if e.reattach() { + return + } + e.SetState(environment.ProcessOfflineState) }() go func() { @@ -99,6 +108,53 @@ func (e *Environment) Attach(ctx context.Context) error { return nil } +// reattach is called when the attached stream ends, and attaches to the +// container again if it is still running, returning true if the container is +// still running. If that keeps happening, or attaching fails, the container is +// terminated rather than left running without being managed. +func (e *Environment) reattach() bool { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + c, err := e.ContainerInspect(ctx) + if err != nil || c.State == nil || !c.State.Running { + return false + } + + e.mu.Lock() + if time.Since(e.reattachedAt) > time.Minute*5 { + e.reattachedAt = time.Now() + e.reattachedCount = 0 + } + e.reattachedCount++ + count := e.reattachedCount + e.mu.Unlock() + + if count <= 3 { + e.log().Warn("lost the attached stream for a running container, attaching again") + err := e.Attach(ctx) + if err == nil { + return true + } + // The container may have only just stopped, in which case it is handled like + // any other stop so that crash detection still applies. + if c, ierr := e.ContainerInspect(ctx); ierr == nil && c.State != nil && !c.State.Running { + return false + } + e.log().WithField("error", err).Error("failed to attach to running container") + } + + e.log().Error("unable to stay attached to running container, terminating it") + // Attaching may have used up the time allowed by ctx, which would leave the + // container running without being managed. + tctx, tcancel := context.WithTimeout(context.Background(), time.Second*30) + defer tcancel() + if err := e.Terminate(tctx, "SIGKILL"); err != nil { + e.log().WithField("error", err).Error("failed to terminate container") + } + return true +} + // InSituUpdate performs an in-place update of the Docker container's resource // limits without actually making any changes to the operational state of the // container. This allows memory, cpu, and IO limitations to be adjusted on the @@ -115,7 +171,7 @@ func (e *Environment) InSituUpdate() error { // to the disk. // // We'll let a boot process make modifications to the container if needed at this point. - if client.IsErrNotFound(err) { + if cerrdefs.IsNotFound(err) { return nil } return errors.Wrap(err, "environment/docker: could not inspect container") @@ -152,7 +208,7 @@ func (e *Environment) Create() error { // container anyways. if _, err := e.ContainerInspect(ctx); err == nil { return nil - } else if !client.IsErrNotFound(err) { + } else if !cerrdefs.IsNotFound(err) { return errors.WrapIf(err, "environment/docker: failed to inspect container") } @@ -210,7 +266,7 @@ func (e *Environment) Create() error { networkMode = container.NetworkMode(networkName) if _, err := e.client.NetworkInspect(ctx, networkName, network.InspectOptions{}); err != nil { - if !client.IsErrNotFound(err) { + if !cerrdefs.IsNotFound(err) { return err } @@ -293,7 +349,7 @@ func (e *Environment) Destroy() error { // exist on the system. We're just a step ahead of ourselves in that case. // // @see https://github.com/pterodactyl/panel/issues/2001 - if err != nil && client.IsErrNotFound(err) { + if err != nil && cerrdefs.IsNotFound(err) { return nil } @@ -304,21 +360,32 @@ func (e *Environment) Destroy() error { // instance. There is no confirmation that this data is sent successfully, only // that it gets pushed into the stdin. func (e *Environment) SendCommand(c string) error { - if !e.IsAttached() { - return errors.Wrap(ErrNotAttached, "environment/docker: cannot send command to container") + // Commands may not contain control characters, other than tabs. + if strings.ContainsFunc(c, func(r rune) bool { + return (r < 0x20 && r != '\t' && r != '\n' && r != '\r') || r == 0x7f + }) { + return errors.New("environment/docker: command contains control characters") } e.mu.RLock() - defer e.mu.RUnlock() + stream := e.stream + stop := e.meta.Stop + e.mu.RUnlock() + if stream == nil { + return errors.Wrap(ErrNotAttached, "environment/docker: cannot send command to container") + } // If the command being processed is the same as the process stop command then we // want to mark the server as entering the stopping state otherwise the process will // stop and Wings will think it has crashed and attempt to restart it. - if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value { + if stop.Type == "command" && c == stop.Value { e.SetState(environment.ProcessStoppingState) } - _, err := e.stream.Conn.Write([]byte(c + "\n")) + // The process may never read its input, so don't wait forever for there to be + // room for the command. + _ = stream.Conn.SetWriteDeadline(time.Now().Add(commandWriteTimeout)) + _, err := stream.Conn.Write([]byte(c + "\n")) return errors.Wrap(err, "environment/docker: could not write to container stream") } diff --git a/environment/docker/environment.go b/environment/docker/environment.go index 4516fccf1..83c848132 100644 --- a/environment/docker/environment.go +++ b/environment/docker/environment.go @@ -3,11 +3,12 @@ package docker import ( "context" "fmt" - "io" "sync" + "time" "emperror.dev/errors" "github.com/apex/log" + cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types" "github.com/docker/docker/client" @@ -45,9 +46,6 @@ type Environment struct { // the running container instance. stream *types.HijackedResponse - // Holds the stats stream used by the polling commands so that we can easily close it out. - stats io.ReadCloser - emitter *events.Bus logCallbackMx sync.Mutex @@ -55,6 +53,11 @@ type Environment struct { // Tracks the environment state. st *system.AtomicString + + // Tracks how often the attached stream has ended while the container kept + // running, guarded by mu. + reattachedAt time.Time + reattachedCount int } // New creates a new base Docker environment. The ID passed through will be the @@ -119,7 +122,7 @@ func (e *Environment) Exists() (bool, error) { if err != nil { // If this error is because the container instance wasn't found via Docker we // can safely ignore the error and just return false. - if client.IsErrNotFound(err) { + if cerrdefs.IsNotFound(err) { return false, nil } return false, err @@ -132,8 +135,7 @@ func (e *Environment) Exists() (bool, error) { // shouldn't be a case that ever happens under correctly developed // circumstances). // -// You can confirm if the instance wasn't found by using client.IsErrNotFound -// from the Docker API. +// You can confirm if the instance wasn't found by using cerrdefs.IsNotFound. // // @see docker/client/errors.go func (e *Environment) IsRunning(ctx context.Context) (bool, error) { @@ -158,7 +160,7 @@ func (e *Environment) ExitState() (uint32, bool, error) { // so that's a mystery that will have to go unsolved. // // @see https://github.com/pterodactyl/panel/issues/2003 - if client.IsErrNotFound(err) { + if cerrdefs.IsNotFound(err) { return 1, false, nil } return 0, false, errors.WrapIf(err, "environment/docker: failed to inspect container") @@ -197,12 +199,13 @@ func (e *Environment) State() string { // can hook into to take their own actions and track their own state based on // the environment. func (e *Environment) SetState(state string) { - if state != environment.ProcessOfflineState && - state != environment.ProcessStartingState && - state != environment.ProcessRunningState && - state != environment.ProcessStoppingState && - state != environment.ProcessBackupState && - state != environment.ProcessRestoringState { + switch state { + case environment.ProcessOfflineState, environment.ProcessStartingState, environment.ProcessRunningState, + environment.ProcessStoppingState, environment.ProcessBackupState, environment.ProcessRestoringState, + // Fork: queue states set by the router once a backup/restore operation + // had to wait for a slot in the backup operation registry. + environment.ProcessBackupQueuedState, environment.ProcessRestoreQueuedState: + default: panic(errors.New(fmt.Sprintf("invalid server state received: %s", state))) } diff --git a/environment/docker/power.go b/environment/docker/power.go index f6c0f7428..4590dec1f 100644 --- a/environment/docker/power.go +++ b/environment/docker/power.go @@ -8,8 +8,8 @@ import ( "emperror.dev/errors" "github.com/apex/log" + cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" "github.com/Rene-Roscher/wings/environment" "github.com/Rene-Roscher/wings/remote" @@ -26,7 +26,7 @@ import ( func (e *Environment) OnBeforeStart(ctx context.Context) error { // Always destroy and re-create the server container to ensure that synced data from the Panel is used. if err := e.client.ContainerRemove(ctx, e.Id, container.RemoveOptions{RemoveVolumes: true}); err != nil { - if !client.IsErrNotFound(err) { + if !cerrdefs.IsNotFound(err) { return errors.WrapIf(err, "environment/docker: failed to remove container during pre-boot") } } @@ -70,7 +70,7 @@ func (e *Environment) Start(ctx context.Context) error { // a nil-pointer when checking c.State below. // // @see https://github.com/pterodactyl/panel/issues/2000 - if !client.IsErrNotFound(err) { + if !cerrdefs.IsNotFound(err) { return errors.WrapIf(err, "environment/docker: failed to inspect container") } } else { @@ -195,7 +195,7 @@ func (e *Environment) Stop(ctx context.Context) error { if err := e.client.ContainerStop(ctx, e.Id, container.StopOptions{Timeout: &timeout}); err != nil { // If the container does not exist just mark the process as stopped and return without // an error. - if client.IsErrNotFound(err) { + if cerrdefs.IsNotFound(err) { e.SetStream(nil) e.SetState(environment.ProcessOfflineState) return nil @@ -237,9 +237,11 @@ func (e *Environment) WaitForStop(ctx context.Context, duration time.Duration, t // We pass through the timed context for this stop action so that if one of the // internal docker calls fails to ever finish before we've exhausted the time limit - // the resources get cleaned up, and the exection is stopped. + // the resources get cleaned up, and the exection is stopped. If the process could + // not be asked to stop for any reason, fall back to terminating it when allowed. if err := e.Stop(tctx); err != nil { - if terminate && errors.Is(err, context.DeadlineExceeded) { + if terminate { + e.log().WithField("error", err).Warn("failed to stop container") return doTermination("stop") } return err @@ -260,7 +262,7 @@ func (e *Environment) WaitForStop(ctx context.Context, duration time.Duration, t case err := <-errChan: // If the error stems from the container not existing there is no point in wasting // CPU time to then try and terminate it. - if err == nil || client.IsErrNotFound(err) { + if err == nil || cerrdefs.IsNotFound(err) { return nil } if terminate { @@ -282,7 +284,7 @@ func (e *Environment) SignalContainer(ctx context.Context, signal string) error if err != nil { // Treat missing containers as an okay error state, means it is obviously // already terminated at this point. - if client.IsErrNotFound(err) { + if cerrdefs.IsNotFound(err) { return nil } return errors.WithStack(err) @@ -302,7 +304,7 @@ func (e *Environment) SignalContainer(ctx context.Context, signal string) error // We set it to stopping than offline to prevent crash detection from being triggered. e.SetState(environment.ProcessStoppingState) - if err := e.client.ContainerKill(ctx, e.Id, signal); err != nil && !client.IsErrNotFound(err) { + if err := e.client.ContainerKill(ctx, e.Id, signal); err != nil && !cerrdefs.IsNotFound(err) { return errors.WithStack(err) } diff --git a/environment/docker/state_test.go b/environment/docker/state_test.go new file mode 100644 index 000000000..03c6bb15a --- /dev/null +++ b/environment/docker/state_test.go @@ -0,0 +1,48 @@ +package docker + +import ( + "testing" + + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/events" + "github.com/Rene-Roscher/wings/system" +) + +// Fork: the backup/restore queue states must be accepted by SetState, otherwise +// a queued backup or restore panics in the router goroutine. +func TestSetStateAcceptsQueuedStates(t *testing.T) { + states := []string{ + environment.ProcessOfflineState, + environment.ProcessStartingState, + environment.ProcessRunningState, + environment.ProcessStoppingState, + environment.ProcessBackupState, + environment.ProcessRestoringState, + environment.ProcessBackupQueuedState, + environment.ProcessRestoreQueuedState, + } + for _, st := range states { + e := &Environment{emitter: events.NewBus(), st: system.NewAtomicString(environment.ProcessOfflineState)} + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("SetState(%q) panicked: %v", st, r) + } + }() + e.SetState(st) + }() + if got := e.State(); got != st { + t.Fatalf("SetState(%q): got state %q", st, got) + } + } +} + +func TestSetStateRejectsUnknownState(t *testing.T) { + e := &Environment{emitter: events.NewBus(), st: system.NewAtomicString(environment.ProcessOfflineState)} + defer func() { + if recover() == nil { + t.Fatal("expected SetState to panic on an unknown state") + } + }() + e.SetState("bogus") +} diff --git a/environment/settings.go b/environment/settings.go index 77f7e7f84..d9143538b 100644 --- a/environment/settings.go +++ b/environment/settings.go @@ -171,21 +171,21 @@ func (v Variables) Get(key string) string { return "" } - switch val.(type) { + switch val := val.(type) { case int: - return strconv.Itoa(val.(int)) + return strconv.Itoa(val) case int32: - return strconv.FormatInt(val.(int64), 10) + return strconv.FormatInt(int64(val), 10) case int64: - return strconv.FormatInt(val.(int64), 10) + return strconv.FormatInt(val, 10) case float32: - return fmt.Sprintf("%f", val.(float32)) + return fmt.Sprintf("%f", val) case float64: - return fmt.Sprintf("%f", val.(float64)) + return fmt.Sprintf("%f", val) case bool: - return strconv.FormatBool(val.(bool)) + return strconv.FormatBool(val) case string: - return val.(string) + return val } // TODO: I think we can add a check for val == nil and return an empty string for those diff --git a/flake.lock b/flake.lock index e87c8a2fc..026252cc1 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1768127708, - "narHash": "sha256-1Sm77VfZh3mU0F5OqKABNLWxOuDeHIlcFjsXeeiPazs=", + "lastModified": 1791456040, + "narHash": "sha256-H83yz/do+hjOO0uujHyH8RgtIQP2v7Mw+cLwcZK/xKc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ffbc9f8cbaacfb331b6017d5a5abb21a492c9a38", + "rev": "e7439b6b14ad3cc35d05608ebca9bce01a25f5f8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a45999387..046b26dd1 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,7 @@ in { devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ - go_1_24 + go_1_26 gofumpt golangci-lint gotools diff --git a/go.mod b/go.mod index 2c1a0e8ab..29a35d150 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/Rene-Roscher/wings -go 1.24.0 +go 1.26.0 -toolchain go1.24.1 +toolchain go1.26.9 require ( emperror.dev/errors v0.8.1 @@ -13,14 +13,17 @@ require ( github.com/apex/log v1.9.0 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/beevik/etree v1.5.0 - github.com/buger/jsonparser v1.1.1 + github.com/buger/jsonparser v1.6.1 github.com/cenkalti/backoff/v4 v4.3.0 + github.com/containerd/errdefs v0.3.0 + github.com/containerd/errdefs/pkg v0.3.0 github.com/creasty/defaults v1.8.0 - github.com/docker/docker v28.3.3+incompatible + github.com/distribution/reference v0.6.0 + github.com/docker/docker v28.5.2+incompatible github.com/docker/go-connections v0.5.0 github.com/fatih/color v1.18.0 github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf - github.com/gabriel-vasile/mimetype v1.4.8 + github.com/gabriel-vasile/mimetype v1.4.15 github.com/gammazero/workerpool v1.1.3 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/gin-gonic/gin v1.10.1 @@ -31,7 +34,7 @@ require ( github.com/iancoleman/strcase v0.3.0 github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/juju/ratelimit v1.0.2 - github.com/klauspost/compress v1.18.0 + github.com/klauspost/compress v1.18.7 github.com/klauspost/pgzip v1.2.6 github.com/magiconair/properties v1.8.9 github.com/mattn/go-colorable v0.1.14 @@ -40,11 +43,13 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.10 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 - github.com/spf13/cobra v1.9.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.46.0 - golang.org/x/sync v0.19.0 - golang.org/x/sys v0.39.0 + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.12.1 + golang.org/x/crypto v0.58.0 + golang.org/x/net v0.60.0 + golang.org/x/sync v0.24.0 + golang.org/x/sys v0.49.0 + golang.org/x/time v0.16.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -61,26 +66,22 @@ require ( github.com/bodgit/windows v1.0.1 // indirect github.com/bytedance/sonic v1.13.1 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect - github.com/containerd/errdefs v0.3.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/gammazero/deque v1.0.0 // indirect github.com/gin-contrib/sse v1.0.0 // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.25.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -108,33 +109,35 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/spf13/afero v1.15.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.15 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect go.uber.org/atomic v1.11.0 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/arch v0.15.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/protobuf v1.36.12 // indirect gotest.tools/v3 v3.0.2 // indirect modernc.org/libc v1.61.13 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 9e2465a94..032c12824 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4 github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buger/jsonparser v1.6.1 h1:I0phFv0PlbLHnM7TZAVjZ2MJ2/eWRTDyuO7GLR98IEs= +github.com/buger/jsonparser v1.6.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -66,7 +66,11 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -92,8 +96,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -108,13 +112,13 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf h1:NrF81UtW8gG2LBGkXFQFqlfNnvMt9WdB46sfdJY4oqc= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -135,8 +139,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -149,8 +153,6 @@ github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0 github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -188,8 +190,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -218,11 +220,10 @@ github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= @@ -306,7 +307,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -317,8 +317,8 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= @@ -333,16 +333,17 @@ github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/i github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -352,8 +353,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -370,42 +372,45 @@ github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= @@ -416,10 +421,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -447,10 +451,9 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -465,13 +468,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -483,10 +484,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -501,7 +501,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -510,13 +509,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -525,12 +524,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -557,15 +556,12 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -595,10 +591,10 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -606,10 +602,10 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/cron/activity_cron.go b/internal/cron/activity_cron.go index 448c966b3..eed956350 100644 --- a/internal/cron/activity_cron.go +++ b/internal/cron/activity_cron.go @@ -30,70 +30,63 @@ func (ac *activityCron) Run(ctx context.Context) error { } defer ac.mu.Store(false) + if err := pruneActivity(ctx, maxStoredActivity); err != nil { + return err + } + + // Keep sending batches until the stored activity has been sent, so that it is + // sent as quickly as it is created. + for i := 0; i < maxBatchesPerRun; i++ { + n, err := ac.sendBatch(ctx) + if err != nil || n < ac.batchSize() { + return err + } + } + return nil +} + +func (ac *activityCron) batchSize() int { + if ac.max <= 0 { + return 100 + } + return ac.max +} + +// sendBatch sends the oldest stored activity to the Panel and returns how many +// rows were read. +func (ac *activityCron) sendBatch(ctx context.Context) (int, error) { var activity []models.Activity tx := database.Instance().WithContext(ctx). Where("event NOT LIKE ?", "server:sftp.%"). - Limit(ac.max). + Order("id ASC"). + Limit(ac.batchSize()). Find(&activity) if tx.Error != nil { - return errors.WithStack(tx.Error) + return 0, errors.WithStack(tx.Error) } if len(activity) == 0 { - return nil + return 0, nil } - // ids to delete from the database. - ids := make([]int, 0, len(activity)) - // activities to send to the panel. - activities := make([]models.Activity, 0, len(activity)) + // ids of activity to delete without sending it. + var invalid []int + items := make([]batchItem, 0, len(activity)) for _, v := range activity { // Delete any activity that has an invalid IP address. This is a fix for // a bug that truncated the last octet of an IPv6 address in the database. if ip := net.ParseIP(v.IP); ip == nil { - ids = append(ids, v.ID) + invalid = append(invalid, v.ID) continue } - activities = append(activities, v) + items = append(items, batchItem{activity: v, ids: []int{v.ID}}) } - // Delete any invalid activies - if len(ids) > 0 { - tx = database.Instance().WithContext(ctx).Where("id IN ?", ids).Delete(&models.Activity{}) - if tx.Error != nil { - return errors.WithStack(tx.Error) - } + done, err := deliver(ctx, ac.manager.Client(), items) + if derr := deleteActivity(ctx, append(invalid, done...)); derr != nil { + return 0, derr } - - if len(activities) == 0 { - return nil + if err != nil { + return 0, errors.WrapIf(err, "cron: failed to send activity events to Panel") } - - if err := ac.manager.Client().SendActivityLogs(ctx, activities); err != nil { - return errors.WrapIf(err, "cron: failed to send activity events to Panel") - } - - ids = make([]int, len(activities)) - for i, v := range activities { - ids[i] = v.ID - } - - // SQLite has a limitation of how many parameters we can specify in a single - // query, so we need to delete the activies in chunks of 32,000 instead of - // all at once. - i := 0 - idsLen := len(ids) - for i < idsLen { - start := i - end := min(i+32000, idsLen) - batchSize := end - start - - tx = database.Instance().WithContext(ctx).Where("id IN ?", ids[start:end]).Delete(&models.Activity{}) - if tx.Error != nil { - return errors.WithStack(tx.Error) - } - - i += batchSize - } - - return nil + return len(activity), nil } diff --git a/internal/cron/activity_delivery_test.go b/internal/cron/activity_delivery_test.go new file mode 100644 index 000000000..87a1e38f4 --- /dev/null +++ b/internal/cron/activity_delivery_test.go @@ -0,0 +1,268 @@ +package cron + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/internal/database" + "github.com/Rene-Roscher/wings/internal/models" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/server" + "github.com/Rene-Roscher/wings/system" +) + +const ( + testServerUuid = "5b8ad3c2-6d1f-4e4a-9a53-0f4f2d7c1e10" + testUserUuid = "9d1e2c3b-4a5f-4b6c-8d7e-1f2a3b4c5d6e" + validIP = "203.0.113.9" + // zoneIP is how an IPv6 address with a zone is recorded for an SFTP session. + zoneIP = "[fe80::1%eth0]:2022" +) + +var dbOnce sync.Once + +// setupDatabase creates the local activity database once for this package and +// removes every stored row before a test runs. +func setupDatabase(t *testing.T) { + t.Helper() + dbOnce.Do(func() { + dir, err := os.MkdirTemp("", "wings-cron-test-") + if err != nil { + panic(err) + } + config.Set(&config.Configuration{ + AuthenticationToken: "test-token", + System: config.SystemConfiguration{RootDirectory: dir}, + }) + if err := database.Initialize(); err != nil { + panic(err) + } + }) + require.NoError(t, database.Instance().Where("1 = 1").Delete(&models.Activity{}).Error) +} + +func insertActivity(t *testing.T, event models.Event, ip, file string) { + t.Helper() + meta := models.ActivityMeta{"files": []string{file}} + if !strings.HasPrefix(string(event), "server:sftp.") { + meta = models.ActivityMeta{"command": file} + } + a := (&models.Activity{ + Server: testServerUuid, + Event: event, + Metadata: meta, + IP: ip, + Timestamp: time.Now(), + }).SetUser(testUserUuid) + require.NoError(t, database.Instance().Create(a).Error) +} + +// fakePanel stands in for POST /api/remote/activity. Like the Panel's ip rule +// (filter_var FILTER_VALIDATE_IP), it rejects the whole batch with a 422 when any +// row has an IP address that does not parse. It also rejects the command +// "refused". +type fakePanel struct { + mu sync.Mutex + attempts [][]string + delivered []string + sizes []int +} + +func (p *fakePanel) ServeHTTP(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + var body struct { + Data []struct { + IP string `json:"ip"` + Metadata map[string]any `json:"metadata"` + } `json:"data"` + } + _ = json.Unmarshal(raw, &body) + + ips := make([]string, 0, len(body.Data)) + valid := true + for _, row := range body.Data { + ips = append(ips, row.IP) + if row.IP != "" && net.ParseIP(row.IP) == nil { + valid = false + } + if row.Metadata["command"] == "refused" { + valid = false + } + } + + p.mu.Lock() + p.attempts = append(p.attempts, ips) + p.sizes = append(p.sizes, len(raw)) + if valid { + p.delivered = append(p.delivered, ips...) + } + p.mu.Unlock() + + if !valid { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(`{"errors":[{"code":"ValidationException","status":"422","detail":"The data.0.ip field must be a valid IP address."}]}`)) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (p *fakePanel) sentIPs() []string { + p.mu.Lock() + defer p.mu.Unlock() + var out []string + for _, a := range p.attempts { + out = append(out, a...) + } + return out +} + +func (p *fakePanel) deliveredIPs() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.delivered...) +} + +// newCrons returns both crons talking to the fake Panel through the real remote +// client. +func newCrons(t *testing.T, panel *fakePanel) (activityCron, sftpCron) { + t.Helper() + srv := httptest.NewServer(panel) + t.Cleanup(srv.Close) + manager := server.NewEmptyManager(remote.New(srv.URL)) + return activityCron{mu: system.NewAtomicBool(false), manager: manager, max: 100}, + sftpCron{mu: system.NewAtomicBool(false), manager: manager, max: 100} +} + +// An SFTP event whose address carries an IPv6 zone is refused by the Panel, so it +// is dropped rather than sent. +func TestSftpActivityWithUnparsableIPIsNotSent(t *testing.T) { + setupDatabase(t) + panel := &fakePanel{} + _, sc := newCrons(t, panel) + insertActivity(t, "server:sftp.write", zoneIP, "/zone.txt") + insertActivity(t, "server:sftp.write", validIP, "/valid.txt") + + require.NoError(t, sc.Run(context.Background())) + + assert.NotContains(t, panel.sentIPs(), "fe80::1%eth0") + assert.Contains(t, panel.deliveredIPs(), validIP) + assert.Zero(t, storedActivity(t)) +} + +// Activity the Panel refuses is dropped so that it does not stop the rest of the +// activity from being sent. +func TestRefusedActivityDoesNotHoldBackOtherActivity(t *testing.T) { + setupDatabase(t) + panel := &fakePanel{} + ac, _ := newCrons(t, panel) + for i := 0; i < 10; i++ { + command := "say hi" + if i == 4 { + command = "refused" + } + insertActivity(t, "server:console.command", validIP, command) + } + + require.NoError(t, ac.Run(context.Background())) + + assert.Len(t, panel.deliveredIPs(), 9) + assert.Zero(t, storedActivity(t)) +} + +// The non-SFTP cron drops activity with an address that does not parse, and +// sends the rest. +func TestActivityWithUnparsableIPIsNotSent(t *testing.T) { + setupDatabase(t) + panel := &fakePanel{} + ac, _ := newCrons(t, panel) + insertActivity(t, "server:console.command", zoneIP, "say hi") + insertActivity(t, "server:console.command", validIP, "say bye") + + require.NoError(t, ac.Run(context.Background())) + + assert.NotContains(t, panel.sentIPs(), "fe80::1%eth0") + assert.Contains(t, panel.deliveredIPs(), validIP) +} + +// All stored activity is sent in one run, rather than one batch per run. +func TestActivityBacklogIsSentInOneRun(t *testing.T) { + setupDatabase(t) + panel := &fakePanel{} + ac, _ := newCrons(t, panel) + insertMany(t, 350, "server:console.command", models.ActivityMeta{"command": "say hi"}) + + require.NoError(t, ac.Run(context.Background())) + + assert.Len(t, panel.deliveredIPs(), 350) + assert.Zero(t, storedActivity(t)) +} + +// SFTP events for one group are split over several events and requests. +func TestSftpEventsAreSentInBoundedRequests(t *testing.T) { + setupDatabase(t) + panel := &fakePanel{} + _, sc := newCrons(t, panel) + path := "/" + strings.Repeat("a", 4000) + insertMany(t, 3000, "server:sftp.write", models.ActivityMeta{"files": []string{path}}) + + require.NoError(t, sc.Run(context.Background())) + + assert.Zero(t, storedActivity(t)) + panel.mu.Lock() + defer panel.mu.Unlock() + require.NotEmpty(t, panel.sizes) + for _, size := range panel.sizes { + assert.LessOrEqual(t, size, maxRequestBytes+(64<<10)) + } +} + +// Once more activity is stored than is allowed, the oldest is deleted. +func TestStoredActivityIsLimited(t *testing.T) { + setupDatabase(t) + insertMany(t, 50, "server:console.command", models.ActivityMeta{"command": "say hi"}) + + require.NoError(t, pruneActivity(context.Background(), 20)) + + assert.Equal(t, int64(20), storedActivity(t)) + var oldest models.Activity + require.NoError(t, database.Instance().Order("id ASC").First(&oldest).Error) + var newest models.Activity + require.NoError(t, database.Instance().Order("id DESC").First(&newest).Error) + assert.Equal(t, 19, newest.ID-oldest.ID) +} + +func insertMany(t *testing.T, n int, event models.Event, meta models.ActivityMeta) { + t.Helper() + batch := make([]models.Activity, 0, n) + for i := 0; i < n; i++ { + batch = append(batch, models.Activity{ + Server: testServerUuid, + Event: event, + Metadata: meta, + IP: validIP, + Timestamp: time.Now(), + }) + } + require.NoError(t, database.Instance().CreateInBatches(&batch, 500).Error) +} + +func storedActivity(t *testing.T) int64 { + t.Helper() + var count int64 + require.NoError(t, database.Instance().Model(&models.Activity{}).Count(&count).Error) + return count +} diff --git a/internal/cron/delivery.go b/internal/cron/delivery.go new file mode 100644 index 000000000..0bde2af01 --- /dev/null +++ b/internal/cron/delivery.go @@ -0,0 +1,120 @@ +package cron + +import ( + "context" + "net/http" + + "emperror.dev/errors" + "github.com/apex/log" + + "github.com/Rene-Roscher/wings/internal/database" + "github.com/Rene-Roscher/wings/internal/models" + "github.com/Rene-Roscher/wings/remote" +) + +const ( + // maxBatchesPerRun is the most batches of activity sent to the Panel in one + // run, so that a large backlog is sent over several runs. + maxBatchesPerRun = 50 + // maxStoredActivity is the most activity kept while it waits to be sent. Once + // there is more, the oldest is deleted. + maxStoredActivity = 100_000 +) + +// batchItem is an activity to send to the Panel along with the IDs of the +// stored rows it was made from. +type batchItem struct { + activity models.Activity + ids []int +} + +// deliver sends the items to the Panel and returns the IDs of the rows that no +// longer need to be kept. If the Panel refuses a batch as invalid, it is split +// to find the items it refuses, which are dropped so that they do not stop the +// rest from ever being sent. +func deliver(ctx context.Context, client remote.Client, items []batchItem) ([]int, error) { + if len(items) == 0 { + return nil, nil + } + activity := make([]models.Activity, len(items)) + for i, item := range items { + activity[i] = item.activity + } + + err := client.SendActivityLogs(ctx, activity) + if err == nil { + var ids []int + for _, item := range items { + ids = append(ids, item.ids...) + } + return ids, nil + } + if !isRefused(err) { + return nil, err + } + if len(items) == 1 { + log.WithField("subsystem", "cron"). + WithField("event", items[0].activity.Event). + WithField("server", items[0].activity.Server). + WithField("error", err). + Warn("dropping activity event refused by the Panel") + return items[0].ids, nil + } + + mid := len(items) / 2 + done, err := deliver(ctx, client, items[:mid]) + if err != nil { + return done, err + } + rest, err := deliver(ctx, client, items[mid:]) + return append(done, rest...), err +} + +// isRefused reports whether the Panel refused a request because of its contents, +// in which case sending it again cannot succeed. +func isRefused(err error) bool { + rerr := remote.AsRequestError(err) + if rerr == nil { + return false + } + switch rerr.StatusCode() { + case http.StatusBadRequest, http.StatusRequestEntityTooLarge, http.StatusUnprocessableEntity: + return true + default: + return false + } +} + +// deleteActivity deletes the stored activity with the given IDs. +func deleteActivity(ctx context.Context, ids []int) error { + // SQLite limits how many parameters a single query can have, so delete the + // rows in chunks. + for start := 0; start < len(ids); start += 32000 { + end := min(start+32000, len(ids)) + tx := database.Instance().WithContext(ctx).Where("id IN ?", ids[start:end]).Delete(&models.Activity{}) + if tx.Error != nil { + return errors.WithStack(tx.Error) + } + } + return nil +} + +// pruneActivity deletes the oldest stored activity once there is more than max. +func pruneActivity(ctx context.Context, max int64) error { + db := database.Instance().WithContext(ctx) + var count int64 + if tx := db.Model(&models.Activity{}).Count(&count); tx.Error != nil { + return errors.WithStack(tx.Error) + } + if count <= max { + return nil + } + excess := count - max + oldest := db.Model(&models.Activity{}).Select("id").Order("id ASC").Limit(int(excess)) + if tx := db.Where("id IN (?)", oldest).Delete(&models.Activity{}); tx.Error != nil { + return errors.WithStack(tx.Error) + } + log.WithField("subsystem", "cron").WithField("deleted", excess). + Warn("deleted the oldest stored activity events because too many are waiting to be sent") + return nil +} diff --git a/internal/cron/sftp_cron.go b/internal/cron/sftp_cron.go index 80d06998e..b3a3daf63 100644 --- a/internal/cron/sftp_cron.go +++ b/internal/cron/sftp_cron.go @@ -2,10 +2,11 @@ package cron import ( "context" + "encoding/json" + "net" "reflect" "emperror.dev/errors" - "gorm.io/gorm" "github.com/Rene-Roscher/wings/internal/database" "github.com/Rene-Roscher/wings/internal/models" @@ -13,6 +14,14 @@ import ( "github.com/Rene-Roscher/wings/system" ) +const ( + // maxFilesPerEvent is the most files listed in one merged SFTP event. Further + // files for the same group start a new event. + maxFilesPerEvent = 500 + // maxRequestBytes is roughly the most activity data sent in one request. + maxRequestBytes = 4 << 20 +) + type sftpCron struct { mu *system.AtomicBool manager *server.Manager @@ -25,12 +34,20 @@ type mapKey struct { IP string Event models.Event Timestamp string + // Part separates the events for a group that has more files than fit in one. + Part int +} + +type eventGroup struct { + activity *models.Activity + ids []int + files int } type eventMap struct { - max int - ids []int - m map[mapKey]*models.Activity + max int + bytes int + m map[mapKey]*eventGroup } // Run executes the SFTP reconciliation cron. This job will pull all of the SFTP specific events @@ -44,67 +61,64 @@ func (sc *sftpCron) Run(ctx context.Context) error { } defer sc.mu.Store(false) - var o int - activity, err := sc.fetchRecords(ctx, o) - if err != nil { - return err - } - o += len(activity) - - events := &eventMap{ - m: map[mapKey]*models.Activity{}, - ids: []int{}, - max: sc.max, + for i := 0; i < maxBatchesPerRun; i++ { + more, err := sc.sendBatch(ctx) + if err != nil || !more { + return err + } } + return nil +} - for { +// sendBatch merges as many stored SFTP events as fit in one request and sends +// them to the Panel. It reports whether there were more events than fit. +func (sc *sftpCron) sendBatch(ctx context.Context) (bool, error) { + events := &eventMap{m: map[mapKey]*eventGroup{}, max: sc.max} + if events.max <= 0 { + events.max = 100 + } + + // ids of activity to delete without sending it. + var invalid []int + var full bool + for o := 0; ; { + activity, err := sc.fetchRecords(ctx, o, events.max) + if err != nil { + return false, err + } if len(activity) == 0 { break } - slen := len(events.ids) + o += len(activity) + + added := false for _, a := range activity { - events.Push(a) - } - if len(events.ids) > slen { - // Execute the query again, we found some events so we want to continue - // with this. Start at the next offset. - activity, err = sc.fetchRecords(ctx, o) - if err != nil { - return errors.WithStack(err) + // The Panel refuses activity with an IP address it cannot parse. + if net.ParseIP(a.IP) == nil { + invalid = append(invalid, a.ID) + continue } - o += len(activity) - } else { + if events.Push(a) { + added = true + } else { + full = true + } + } + // Stop once a page adds nothing, since every later event belongs in + // another request. + if !added { break } } - if len(events.m) == 0 { - return nil - } - if err := sc.manager.Client().SendActivityLogs(ctx, events.Elements()); err != nil { - return errors.Wrap(err, "failed to send sftp activity logs to Panel") + done, err := deliver(ctx, sc.manager.Client(), events.Items()) + if derr := deleteActivity(ctx, append(invalid, done...)); derr != nil { + return false, derr } - - // SQLite has a limitation of how many parameters we can specify in a single - // query, so we need to delete the activies in chunks of 32,000 instead of - // all at once. - i := 0 - idsLen := len(events.ids) - var tx *gorm.DB - for i < idsLen { - start := i - end := min(i+32000, idsLen) - batchSize := end - start - - tx = database.Instance().WithContext(ctx).Where("id IN ?", events.ids[start:end]).Delete(&models.Activity{}) - if tx.Error != nil { - return errors.WithStack(tx.Error) - } - - i += batchSize + if err != nil { + return false, errors.Wrap(err, "failed to send sftp activity logs to Panel") } - - return nil + return full && len(done) > 0, nil } // fetchRecords returns a group of activity events starting at the given offset. This is used @@ -112,12 +126,12 @@ func (sc *sftpCron) Run(ctx context.Context) error { // fill up our request to the given maximum. This is due to the fact that this cron merges any // activity that line up across user, server, ip, and event into a single activity record when // sending the data to the Panel. -func (sc *sftpCron) fetchRecords(ctx context.Context, offset int) (activity []models.Activity, err error) { +func (sc *sftpCron) fetchRecords(ctx context.Context, offset int, limit int) (activity []models.Activity, err error) { tx := database.Instance().WithContext(ctx). Where("event LIKE ?", "server:sftp.%"). Order("event DESC"). Offset(offset). - Limit(sc.max). + Limit(limit). Find(&activity) if tx.Error != nil { err = errors.WithStack(tx.Error) @@ -126,16 +140,20 @@ func (sc *sftpCron) fetchRecords(ctx context.Context, offset int) (activity []mo } // Push adds an activity to the event mapping, or de-duplicates it and merges the files metadata -// into the existing entity that exists. -func (em *eventMap) Push(a models.Activity) { - m := em.forActivity(a) - // If no activity entity is returned we've hit the cap for the number of events to - // send along to the Panel. Just skip over this record and we'll account for it in - // the next iteration. - if m == nil { - return - } - em.ids = append(em.ids, a.ID) +// into the existing entity that exists. It returns false if the activity does not fit in this +// request. +func (em *eventMap) Push(a models.Activity) bool { + size := activitySize(a) + if em.bytes > 0 && em.bytes+size > maxRequestBytes { + return false + } + g := em.forActivity(a) + if g == nil { + return false + } + em.bytes += size + g.ids = append(g.ids, a.ID) + m := g.activity // Always reduce this to the first timestamp that was recorded for the set // of events, and not if a.Timestamp.Before(m.Timestamp) { @@ -145,29 +163,33 @@ func (em *eventMap) Push(a models.Activity) { if s, ok := a.Metadata["files"]; ok { v := reflect.ValueOf(s) if v.Kind() != reflect.Slice || v.IsNil() { - return + return true } for i := 0; i < v.Len(); i++ { list = append(list, v.Index(i).Interface()) } + g.files += v.Len() // You must set it again at the end of the process, otherwise you've only updated the file // slice in this one loop since it isn't passed by reference. This is just shorter than having // to explicitly keep casting it to the slice. m.Metadata["files"] = list } + return true } -// Elements returns the finalized activity models. -func (em *eventMap) Elements() (out []models.Activity) { - for _, v := range em.m { - out = append(out, *v) +// Items returns the merged events with the IDs of the activity they were made +// from. +func (em *eventMap) Items() []batchItem { + out := make([]batchItem, 0, len(em.m)) + for _, g := range em.m { + out = append(out, batchItem{activity: *g.activity, ids: g.ids}) } - return + return out } // forActivity returns an event entity from our map which allows existing matches to be // updated with additional files. -func (em *eventMap) forActivity(a models.Activity) *models.Activity { +func (em *eventMap) forActivity(a models.Activity) *eventGroup { key := mapKey{ User: a.User.String, Server: a.Server, @@ -176,8 +198,15 @@ func (em *eventMap) forActivity(a models.Activity) *models.Activity { // We group by the minute, don't care about the seconds for this logic. Timestamp: a.Timestamp.Format("2006-01-02_15:04"), } - if v, ok := em.m[key]; ok { - return v + for { + v, ok := em.m[key] + if !ok { + break + } + if v.files < maxFilesPerEvent { + return v + } + key.Part++ } // Cap the size of the events map at the defined maximum events to send to the Panel. Just // return nil and let the caller handle it. @@ -190,6 +219,13 @@ func (em *eventMap) forActivity(a models.Activity) *models.Activity { v.Metadata = models.ActivityMeta{ "files": make([]interface{}, 0), } - em.m[key] = &v - return &v + g := &eventGroup{activity: &v} + em.m[key] = g + return g +} + +// activitySize estimates how much an activity adds to a request. +func activitySize(a models.Activity) int { + b, _ := json.Marshal(a.Metadata) + return len(b) + 256 } diff --git a/internal/network/address.go b/internal/network/address.go new file mode 100644 index 000000000..f1849feef --- /dev/null +++ b/internal/network/address.go @@ -0,0 +1,54 @@ +// Package network contains helpers for deciding which network destinations +// Wings may connect to on behalf of a user. +package network + +import ( + "net/netip" +) + +// nat64Prefix is the well-known NAT64 prefix, which embeds an IPv4 address in +// the last 32 bits of the IPv6 address. +var nat64Prefix = netip.MustParsePrefix("64:ff9b::/96") + +// specialPurposePrefixes are the special-purpose ranges from the IANA IPv4 and +// IPv6 registries that are not caught by the other checks in IsInternal. +var specialPurposePrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), // "This network" + netip.MustParsePrefix("100.64.0.0/10"), // Shared address space (carrier-grade NAT) + netip.MustParsePrefix("192.0.0.0/24"), // IETF protocol assignments + netip.MustParsePrefix("192.0.2.0/24"), // Documentation (TEST-NET-1) + netip.MustParsePrefix("192.88.99.0/24"), // 6to4 relay anycast + netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking + netip.MustParsePrefix("198.51.100.0/24"), // Documentation (TEST-NET-2) + netip.MustParsePrefix("203.0.113.0/24"), // Documentation (TEST-NET-3) + netip.MustParsePrefix("240.0.0.0/4"), // Reserved + netip.MustParsePrefix("::/96"), // IPv4-compatible (deprecated) + netip.MustParsePrefix("64:ff9b:1::/48"), // Local-use IPv4/IPv6 translation + netip.MustParsePrefix("100::/64"), // Discard-only + netip.MustParsePrefix("2001::/23"), // IETF protocol assignments (including Teredo) + netip.MustParsePrefix("2001:db8::/32"), // Documentation + netip.MustParsePrefix("2002::/16"), // 6to4 + netip.MustParsePrefix("3fff::/20"), // Documentation + netip.MustParsePrefix("5f00::/16"), // Segment routing (SRv6) SIDs + netip.MustParsePrefix("fec0::/10"), // Site-local (deprecated) +} + +// IsInternal reports whether the address is anything other than a globally +// reachable unicast address. +func IsInternal(addr netip.Addr) bool { + // Prefixes never contain an address with a zone, so it has to be removed. + addr = addr.Unmap().WithZone("") + if !addr.IsValid() || !addr.IsGlobalUnicast() || addr.IsPrivate() { + return true + } + if nat64Prefix.Contains(addr) { + b := addr.As16() + return IsInternal(netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]})) + } + for _, p := range specialPurposePrefixes { + if p.Contains(addr) { + return true + } + } + return false +} diff --git a/internal/network/address_test.go b/internal/network/address_test.go new file mode 100644 index 000000000..426d3b251 --- /dev/null +++ b/internal/network/address_test.go @@ -0,0 +1,38 @@ +package network + +import ( + "net/netip" + "testing" +) + +func TestIsInternal(t *testing.T) { + internal := []string{ + "0.0.0.0", "0.1.2.3", "10.0.0.1", "100.64.0.1", "100.127.255.254", "127.0.0.1", "127.255.255.254", + "169.254.169.254", "172.16.0.1", "172.31.255.254", "192.0.0.8", "192.0.2.1", "192.88.99.1", + "192.168.1.1", "198.18.0.1", "198.19.255.254", "198.51.100.1", "203.0.113.1", "224.0.0.1", + "239.255.255.250", "240.0.0.1", "255.255.255.255", + "::", "::1", "::127.0.0.1", "::ffff:127.0.0.1", "::ffff:10.0.0.1", "::ffff:100.64.0.1", + "64:ff9b::a00:1", "64:ff9b::7f00:1", "64:ff9b:1::1", "100::1", "2001::1", "2001:db8::1", + "2002:a00:1::1", "3fff::1", "5f00::1", "fc00::1", "fd00::1", "fe80::1", "fec0::1", "ff02::1", + "fe80::1%eth0", "fec0::1%eth0", "64:ff9b::7f00:1%lo", "2002:a00:1::1%eth0", "100::1%eth0", "fd00::1%eth0", + } + external := []string{ + "1.1.1.1", "8.8.8.8", "100.63.255.255", "100.128.0.0", "172.15.255.255", "172.32.0.0", + "198.17.255.255", "198.20.0.0", "::ffff:1.1.1.1", "64:ff9b::808:808", "2606:4700:4700::1111", + "2a00:1450:4001::1", + } + + for _, v := range internal { + if !IsInternal(netip.MustParseAddr(v)) { + t.Errorf("expected %s to be internal", v) + } + } + for _, v := range external { + if IsInternal(netip.MustParseAddr(v)) { + t.Errorf("expected %s to not be internal", v) + } + } + if !IsInternal(netip.Addr{}) { + t.Error("expected the zero address to be internal") + } +} diff --git a/internal/network/allowlist.go b/internal/network/allowlist.go new file mode 100644 index 000000000..69e118c6a --- /dev/null +++ b/internal/network/allowlist.go @@ -0,0 +1,30 @@ +package network + +import ( + "net/netip" + "strings" +) + +// IsAllowed reports whether a connection to the address, resolved from the +// host, is allowed by an allowlist configured by the operator. Entries may be +// host names, IP addresses or CIDR ranges. +func IsAllowed(allowlist []string, host string, addr netip.Addr) bool { + host = strings.TrimSuffix(strings.ToLower(host), ".") + addr = addr.Unmap() + for _, entry := range allowlist { + entry = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(entry)), ".") + if entry == "" { + continue + } + if entry == host { + return true + } + if allowedAddr, err := netip.ParseAddr(entry); err == nil && allowedAddr.Unmap() == addr { + return true + } + if prefix, err := netip.ParsePrefix(entry); err == nil && prefix.Contains(addr) { + return true + } + } + return false +} diff --git a/internal/network/allowlist_test.go b/internal/network/allowlist_test.go new file mode 100644 index 000000000..3329c7320 --- /dev/null +++ b/internal/network/allowlist_test.go @@ -0,0 +1,31 @@ +package network + +import ( + "net/netip" + "testing" +) + +func TestIsAllowed(t *testing.T) { + allowlist := []string{" Internal.Example. ", "10.1.2.3", "100.64.0.0/10", ""} + tests := []struct { + host string + addr string + allowed bool + }{ + {"internal.example", "10.0.0.1", true}, + {"INTERNAL.EXAMPLE.", "10.0.0.1", true}, + {"other.example", "10.1.2.3", true}, + {"other.example", "::ffff:10.1.2.3", true}, + {"other.example", "100.100.2.148", true}, + {"other.example", "10.0.0.1", false}, + {"", "192.168.1.1", false}, + } + for _, tt := range tests { + if got := IsAllowed(allowlist, tt.host, netip.MustParseAddr(tt.addr)); got != tt.allowed { + t.Errorf("IsAllowed(%q, %s) = %v, expected %v", tt.host, tt.addr, got, tt.allowed) + } + } + if IsAllowed(nil, "internal.example", netip.MustParseAddr("10.0.0.1")) { + t.Error("expected nothing to be allowed by an empty allowlist") + } +} diff --git a/internal/network/limit_listener.go b/internal/network/limit_listener.go new file mode 100644 index 000000000..799d3d2d4 --- /dev/null +++ b/internal/network/limit_listener.go @@ -0,0 +1,63 @@ +package network + +import ( + "net" + "sync" + "sync/atomic" + "time" + + "github.com/apex/log" + "golang.org/x/net/netutil" +) + +// limitWarnInterval is the minimum time between warnings that a listener has +// reached its limit. +const limitWarnInterval = time.Minute + +// LimitListener returns a listener that accepts at most n connections at once, +// like netutil.LimitListener. Once the limit is reached, new connections wait +// for an open one to close without anything else being logged, so this logs a +// warning, at most once a minute, whenever the limit is reached. +func LimitListener(l net.Listener, n int, name string) net.Listener { + return &limitListener{Listener: netutil.LimitListener(l, n), limit: int64(n), name: name} +} + +type limitListener struct { + net.Listener + limit int64 + name string + open atomic.Int64 + lastWarn atomic.Int64 +} + +func (l *limitListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + if l.open.Add(1) >= l.limit { + l.warn() + } + return &limitListenerConn{Conn: c, l: l}, nil +} + +func (l *limitListener) warn() { + now := time.Now().UnixNano() + last := l.lastWarn.Load() + if (last != 0 && now-last < int64(limitWarnInterval)) || !l.lastWarn.CompareAndSwap(last, now) { + return + } + log.WithField("limit", l.limit).Warn(l.name + ": reached the maximum number of open connections, new connections will wait until others close (raise the open file limit for Wings to allow more)") +} + +type limitListenerConn struct { + net.Conn + l *limitListener + once sync.Once +} + +func (c *limitListenerConn) Close() error { + err := c.Conn.Close() + c.once.Do(func() { c.l.open.Add(-1) }) + return err +} diff --git a/internal/network/limit_listener_test.go b/internal/network/limit_listener_test.go new file mode 100644 index 000000000..588f08a49 --- /dev/null +++ b/internal/network/limit_listener_test.go @@ -0,0 +1,77 @@ +package network + +import ( + "net" + "strings" + "testing" + "time" + + "github.com/apex/log" + "github.com/apex/log/handlers/memory" +) + +// Connections waiting on the limit are otherwise not logged at all, which left +// operators with nothing to go on when a node stopped accepting connections. +func TestLimitListenerWarnsWhenTheLimitIsReached(t *testing.T) { + handler := memory.New() + logger := log.Log.(*log.Logger) + previous := logger.Handler + logger.Handler = handler + t.Cleanup(func() { logger.Handler = previous }) + + inner, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + l := LimitListener(inner, 2, "test").(*limitListener) + defer l.Close() + + var accepted []net.Conn + for i := 0; i < 2; i++ { + client, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + c, err := l.Accept() + if err != nil { + t.Fatal(err) + } + accepted = append(accepted, c) + if i == 0 && len(handler.Entries) != 0 { + t.Fatalf("expected no warning below the limit, got %q", handler.Entries[0].Message) + } + } + if len(handler.Entries) != 1 || !strings.Contains(handler.Entries[0].Message, "test: reached the maximum number of open connections") { + t.Fatalf("expected one warning once the limit was reached, got %v", handler.Entries) + } + + // Closing a connection more than once only frees its slot once. + _ = accepted[0].Close() + _ = accepted[0].Close() + if n := l.open.Load(); n != 1 { + t.Fatalf("expected one open connection, got %d", n) + } + + // The connection that was closed makes room for another one. + client, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + done := make(chan net.Conn, 1) + go func() { + c, _ := l.Accept() + done <- c + }() + select { + case c := <-done: + _ = c.Close() + case <-time.After(5 * time.Second): + t.Fatal("expected a connection to be accepted after another was closed") + } + // The warning is only logged once a minute. + if len(handler.Entries) != 1 { + t.Fatalf("expected the warning to not be repeated within a minute, got %d entries", len(handler.Entries)) + } +} diff --git a/internal/ufs/fs_chmod_test.go b/internal/ufs/fs_chmod_test.go new file mode 100644 index 000000000..4318837ad --- /dev/null +++ b/internal/ufs/fs_chmod_test.go @@ -0,0 +1,47 @@ +//go:build unix + +package ufs + +import ( + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +// The fallback used where fchmodat2 is unavailable changes the mode of a file +// and refuses to follow a symbolic link. +func TestChmodWithoutFchmodat2(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.WriteFile(outside, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "file.txt"), nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + + dirfd, err := unix.Open(root, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(dirfd) + + if err := fchmodNoFollow(dirfd, "file.txt", 0o600); err != nil { + t.Fatalf("expected the mode to be changed, got %v", err) + } + if st, err := os.Stat(filepath.Join(root, "file.txt")); err != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("expected mode 0600, got %v (%v)", st.Mode().Perm(), err) + } + + if err := fchmodNoFollow(dirfd, "link", 0o600); err == nil { + t.Fatal("expected changing the mode of a symlink to fail") + } + if st, err := os.Stat(outside); err != nil || st.Mode().Perm() != 0o644 { + t.Fatalf("expected the link target to be unchanged, got %v (%v)", st.Mode().Perm(), err) + } +} diff --git a/internal/ufs/fs_quota.go b/internal/ufs/fs_quota.go index 6a3d6b271..dc2170976 100644 --- a/internal/ufs/fs_quota.go +++ b/internal/ufs/fs_quota.go @@ -6,6 +6,8 @@ package ufs import ( "math" "sync/atomic" + + "golang.org/x/sys/unix" ) // Quota is a wrapper around [*UnixFS] that provides the ability to limit the @@ -140,13 +142,7 @@ func (fs *Quota) Remove(name string) error { return err } - // Don't reduce the quota's usage as `name` is not a regular file. - if !s.Mode().IsRegular() { - return nil - } - - // Remove the size of the deleted file from the quota usage. - fs.Add(-s.Size()) + fs.Add(-freedBy(s)) return nil } @@ -182,9 +178,23 @@ func (fs *Quota) removeAll(path string) error { func (fs *Quota) unlinkat(dirfd int, name string, flags int) error { if flags == 0 { s, err := fs.Lstatat(dirfd, name) - if err == nil && s.Mode().IsRegular() { - fs.Add(-s.Size()) + if err == nil { + fs.Add(-freedBy(s)) } } return fs.UnixFS.unlinkat(dirfd, name, flags) } + +// freedBy returns the number of bytes removing the name described by s frees. +// Only regular files count towards the usage. A file with other names (hard +// links) keeps its data and is only counted once, so removing one of several +// names frees nothing. +func freedBy(s FileInfo) int64 { + if !s.Mode().IsRegular() { + return 0 + } + if st, ok := s.Sys().(*unix.Stat_t); ok && st.Nlink > 1 { + return 0 + } + return s.Size() +} diff --git a/internal/ufs/fs_unix.go b/internal/ufs/fs_unix.go index b3197bfd6..3b7a2fb88 100644 --- a/internal/ufs/fs_unix.go +++ b/internal/ufs/fs_unix.go @@ -90,7 +90,36 @@ func (fs *UnixFS) Chmodat(dirfd int, name string, mode FileMode) error { } func (fs *UnixFS) fchmodat(op string, dirfd int, name string, mode FileMode) error { - return ensurePathError(unix.Fchmodat(dirfd, name, uint32(mode), AT_SYMLINK_NOFOLLOW), op, name) + err := unix.Fchmodat(dirfd, name, uint32(mode), AT_SYMLINK_NOFOLLOW) + if err == unix.EOPNOTSUPP { + // Kernels before 6.6 do not support fchmodat2, which is needed to pass + // AT_SYMLINK_NOFOLLOW. + err = fchmodNoFollow(dirfd, name, uint32(mode)) + } + return ensurePathError(err, op, name) +} + +// fchmodNoFollow changes the mode of name without following it if it is a +// symbolic link, without needing fchmodat2. +func fchmodNoFollow(dirfd int, name string, mode uint32) error { + fd, err := unix.Openat(dirfd, name, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) + if err != nil { + return err + } + defer unix.Close(fd) + + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return err + } + // Linux does not support changing the mode of a symbolic link. + if st.Mode&unix.S_IFMT == unix.S_IFLNK { + return unix.EOPNOTSUPP + } + // The descriptor was opened with O_PATH, which fchmod does not accept, so + // change the mode through its entry in /proc, which refers to the file that + // was opened. + return unix.Chmod("/proc/self/fd/"+strconv.Itoa(fd), mode) } // Chown changes the numeric uid and gid of the named file. @@ -510,17 +539,46 @@ func (fs *UnixFS) Rename(oldpath, newpath string) error { return nil } -// Stat returns a FileInfo describing the named file. +// Stat returns a FileInfo describing the named file. If the file is a symbolic +// link, the file it refers to is described, as long as that file is inside the +// filesystem. // // If there is an error, it will be of type *PathError. func (fs *UnixFS) Stat(name string) (FileInfo, error) { - return fs._fstat("stat", name, 0) + info, err := fs.Lstat(name) + if err != nil || info.Mode()&ModeSymlink == 0 { + return info, err + } + return fs.statFollow(name) } -// Statat is like Stat but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Statat(dirfd int, name string) (FileInfo, error) { - return fs._fstatat("statat", dirfd, name, 0) +// statFollow describes the file the symbolic link name refers to, resolving the +// link from the root of the filesystem so that it cannot lead outside of it. +func (fs *UnixFS) statFollow(name string) (FileInfo, error) { + rel, err := fs.unsafePath(name) + if err != nil { + return nil, err + } + rootfd, err := fs._openat(AT_EMPTY_PATH, fs.basePath, O_DIRECTORY|O_RDONLY, 0) + if err != nil { + return nil, err + } + defer unix.Close(rootfd) + + fd, err := fs.openatResolved(rootfd, rel, unix.O_PATH, 0) + if err != nil { + return nil, err + } + defer unix.Close(fd) + + var s fileStat + if err := ignoringEINTR(func() error { + return unix.Fstat(fd, &s.sys) + }); err != nil { + return nil, ensurePathError(err, "stat", name) + } + fillFileStatFromSys(&s, filepath.Base(rel)) + return &s, nil } // Lstat returns a FileInfo describing the named file. @@ -533,16 +591,32 @@ func (fs *UnixFS) Lstat(name string) (FileInfo, error) { return fs._fstat("lstat", name, AT_SYMLINK_NOFOLLOW) } +// Readlinkat returns the target of the symbolic link name in the directory +// dirfd. +func (fs *UnixFS) Readlinkat(dirfd int, name string) (string, error) { + for size := 256; ; size *= 2 { + b := make([]byte, size) + var n int + err := ignoringEINTR(func() error { + var err error + n, err = unix.Readlinkat(dirfd, name, b) + return err + }) + if err != nil { + return "", ensurePathError(err, "readlinkat", name) + } + if n < size { + return string(b[:n]), nil + } + } +} + // Lstatat is like Lstat but allows passing an existing directory file // descriptor rather than needing to resolve one. func (fs *UnixFS) Lstatat(dirfd int, name string) (FileInfo, error) { return fs._fstatat("lstatat", dirfd, name, AT_SYMLINK_NOFOLLOW) } -func (fs *UnixFS) fstat(name string, flags int) (FileInfo, error) { - return fs._fstat("fstat", name, flags) -} - func (fs *UnixFS) _fstat(op string, name string, flags int) (FileInfo, error) { dirfd, name, closeFd, err := fs.safePath(name) defer closeFd() @@ -552,10 +626,6 @@ func (fs *UnixFS) _fstat(op string, name string, flags int) (FileInfo, error) { return fs._fstatat(op, dirfd, name, flags) } -func (fs *UnixFS) fstatat(dirfd int, name string, flags int) (FileInfo, error) { - return fs._fstatat("fstatat", dirfd, name, flags) -} - func (fs *UnixFS) _fstatat(op string, dirfd int, name string, flags int) (FileInfo, error) { var s fileStat if err := ignoringEINTR(func() error { @@ -658,6 +728,19 @@ func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, if flag&O_NOFOLLOW == 0 { flag |= O_NOFOLLOW } + return fs.openatResolved(dirfd, name, flag, mode) +} + +// openatResolved is openat without O_NOFOLLOW being added, so that a symbolic +// link as the last element of name is followed. Any symbolic link must still +// resolve to a path beneath dirfd. +func (fs *UnixFS) openatResolved(dirfd int, name string, flag int, mode FileMode) (int, error) { + // Open everything without blocking and refuse anything that is not a regular + // file or a directory. Paths opened with O_PATH are never read or written. + checkType := flag&unix.O_PATH == 0 + if checkType { + flag |= unix.O_NONBLOCK + } var fd int for { @@ -677,6 +760,13 @@ func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, return 0, err } + if checkType { + if err := checkOpenedType(fd, name); err != nil { + _ = unix.Close(fd) + return 0, err + } + } + // If we are using openat2, we don't need the additional security checks. if fs.useOpenat2 { return fd, nil @@ -684,12 +774,23 @@ func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, // If we are not using openat2, do additional path checking. This assumes // that openat2 is using `RESOLVE_BENEATH` to avoid the same security - // issue. + // issue. The file is closed if the check fails, since callers only use the + // file descriptor when there is no error. + if err := fs.checkOpenedPath(fd, name); err != nil { + _ = unix.Close(fd) + return 0, err + } + return fd, nil +} + +// checkOpenedPath returns an error if the file opened as fd, without the +// protection of openat2, resolved to a path outside of the filesystem. +func (fs *UnixFS) checkOpenedPath(fd int, name string) error { var finalPath string finalPath, err := filepath.EvalSymlinks(filepath.Join("/proc/self/fd/", strconv.Itoa(fd))) if err != nil { if !errors.Is(err, ErrNotExist) { - return fd, fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) + return fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) } // The target of one of the symlinks (EvalSymlinks is recursive) @@ -697,7 +798,7 @@ func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, // that for further validation instead. var pErr *PathError if !errors.As(err, &pErr) { - return fd, fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) + return fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) } // Update the final path to whatever directory or path didn't exist while @@ -713,15 +814,31 @@ func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, if fs.useOpenat2 { op = "openat2" } - return fd, &PathError{ + return &PathError{ Op: op, Path: name, Err: ErrBadPathResolution, } } - // Return the file descriptor and any potential error. - return fd, err + return err +} + +// checkOpenedType returns an error if the file descriptor opened without +// blocking is not a regular file or a directory, and otherwise switches it back +// to blocking mode. +func checkOpenedType(fd int, name string) error { + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return convertErrorType(&PathError{Op: "fstat", Path: name, Err: err}) + } + if t := st.Mode & unix.S_IFMT; t != unix.S_IFREG && t != unix.S_IFDIR { + return &PathError{Op: "openat", Path: name, Err: ErrNotRegular} + } + if err := unix.SetNonblock(fd, false); err != nil { + return convertErrorType(&PathError{Op: "fcntl", Path: name, Err: err}) + } + return nil } // _openat is a wrapper around unix.Openat. This method should never be directly @@ -735,12 +852,12 @@ func (fs *UnixFS) _openat(dirfd int, name string, flag int, mode uint32) (int, e } // O_LARGEFILE is set by Openat for us automatically. fd, err := unix.Openat(dirfd, name, flag, mode) - switch { - case err == nil: + switch err { + case nil: return fd, nil - case err == unix.EINTR: + case unix.EINTR: return fd, err - case err == unix.EAGAIN: + case unix.EAGAIN: return fd, err default: return fd, ensurePathError(err, "openat", name) @@ -774,12 +891,12 @@ func (fs *UnixFS) _openat2(dirfd int, name string, flag, mode uint64) (int, erro // This is why using Openat2 over Openat is preferred if available. Resolve: unix.RESOLVE_BENEATH, }) - switch { - case err == nil: + switch err { + case nil: return fd, nil - case err == unix.EINTR: + case unix.EINTR: return fd, err - case err == unix.EAGAIN: + case unix.EAGAIN: return fd, err default: return fd, ensurePathError(err, "openat2", name) diff --git a/internal/ufs/link_containment_test.go b/internal/ufs/link_containment_test.go new file mode 100644 index 000000000..da46efa95 --- /dev/null +++ b/internal/ufs/link_containment_test.go @@ -0,0 +1,177 @@ +//go:build unix + +package ufs_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// newLinkedRoot returns a filesystem whose root holds symlinks to a directory and +// a file next to the root. +func newLinkedRoot(t *testing.T, openat2 bool) (*ufs.UnixFS, string) { + t.Helper() + + base := t.TempDir() + root := filepath.Join(base, "root") + outside := filepath.Join(base, "outside") + require.NoError(t, os.Mkdir(root, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(outside, "sub"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(outside, "file.txt"), []byte("outside"), 0o644)) + require.NoError(t, os.Symlink(outside, filepath.Join(root, "link"))) + require.NoError(t, os.Symlink(filepath.Join(outside, "file.txt"), filepath.Join(root, "filelink"))) + + fs, err := ufs.NewUnixFS(root, openat2) + require.NoError(t, err) + + return fs, outside +} + +// openDescriptorCount returns the number of file descriptors this process holds. +func openDescriptorCount(t *testing.T) int { + t.Helper() + + entries, err := os.ReadDir("/proc/self/fd") + require.NoError(t, err) + + return len(entries) +} + +// Stat does not describe a file outside the root that a symlink points at. +func TestStatDoesNotFollowLinksOutsideRoot(t *testing.T) { + for _, mode := range []struct { + name string + openat2 bool + }{{"openat2", true}, {"openat", false}} { + t.Run(mode.name, func(t *testing.T) { + fs, _ := newLinkedRoot(t, mode.openat2) + + for _, name := range []string{"link", "filelink"} { + _, err := fs.Stat(name) + require.ErrorIs(t, err, ufs.ErrBadPathResolution, name) + } + }) + } +} + +// Stat still follows symlinks that stay inside the root. +func TestStatFollowsLinksInsideRoot(t *testing.T) { + for _, mode := range []struct { + name string + openat2 bool + }{{"openat2", true}, {"openat", false}} { + t.Run(mode.name, func(t *testing.T) { + fs, _ := newLinkedRoot(t, mode.openat2) + root := fs.BasePath() + require.NoError(t, os.MkdirAll(filepath.Join(root, "shared", "plugins"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "shared", "data.txt"), []byte("12345"), 0o644)) + require.NoError(t, os.Mkdir(filepath.Join(root, "server"), 0o755)) + require.NoError(t, os.Symlink("../shared/plugins", filepath.Join(root, "server", "plugins"))) + require.NoError(t, os.Symlink("shared/data.txt", filepath.Join(root, "rel-link"))) + + st, err := fs.Stat("server/plugins") + require.NoError(t, err) + require.True(t, st.IsDir()) + + st, err = fs.Stat("rel-link") + require.NoError(t, err) + require.Equal(t, int64(5), st.Size()) + }) + } +} + +// A parent directory that resolves outside the root is rejected, and its +// descriptor is closed. +func TestRejectedParentReleasesDescriptor(t *testing.T) { + fs, _ := newLinkedRoot(t, false) + + before := openDescriptorCount(t) + for range 100 { + _, err := fs.Lstat("link/sub/file.txt") + require.ErrorIs(t, err, ufs.ErrBadPathResolution) + } + require.LessOrEqual(t, openDescriptorCount(t)-before, 5) +} + +// Touch through a link to a directory outside the root fails. +func TestTouchThroughLinkCreatesNothingOutsideRoot(t *testing.T) { + for _, mode := range []struct { + name string + openat2 bool + }{{"openat2", true}, {"openat", false}} { + t.Run(mode.name, func(t *testing.T) { + fs, outside := newLinkedRoot(t, mode.openat2) + + _, err := fs.Touch("link/sub/new.txt", ufs.O_RDWR|ufs.O_TRUNC, 0o644) + require.Error(t, err) + require.NoFileExists(t, filepath.Join(outside, "sub", "new.txt")) + }) + } +} + +// Open and Touch through a link to a directory outside the root fail. +func TestOpenAndTruncateThroughLinkLeaveOutsideFileUnchanged(t *testing.T) { + for _, mode := range []struct { + name string + openat2 bool + }{{"openat2", true}, {"openat", false}} { + t.Run(mode.name, func(t *testing.T) { + fs, outside := newLinkedRoot(t, mode.openat2) + require.NoError(t, os.WriteFile(filepath.Join(outside, "sub", "existing.txt"), []byte("outside"), 0o644)) + + _, err := fs.Open("link/sub/existing.txt") + require.Error(t, err) + + _, err = fs.Touch("link/sub/existing.txt", ufs.O_RDWR|ufs.O_TRUNC, 0o644) + require.Error(t, err) + + content, err := os.ReadFile(filepath.Join(outside, "sub", "existing.txt")) + require.NoError(t, err) + require.Equal(t, "outside", string(content)) + }) + } +} + +// MkdirAll through a link to a directory outside the root fails. +func TestMkdirAllThroughLinkCreatesNothingOutsideRoot(t *testing.T) { + for _, mode := range []struct { + name string + openat2 bool + }{{"openat2", true}, {"openat", false}} { + t.Run(mode.name, func(t *testing.T) { + fs, outside := newLinkedRoot(t, mode.openat2) + + _, err := fs.MkdirAll("link/newdir/child", 0o755) + require.Error(t, err) + require.NoDirExists(t, filepath.Join(outside, "newdir")) + }) + } +} + +// Changing the mode of a link leaves the file it points at unchanged. +func TestChmodThroughLinkLeavesOutsideFileUnchanged(t *testing.T) { + fs, outside := newLinkedRoot(t, true) + + _ = fs.Chmod("filelink", 0o600) + + st, err := os.Stat(filepath.Join(outside, "file.txt")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o644), st.Mode().Perm()) +} + +// RemoveAll through a link only removes the link. +func TestRemoveAllThroughLinkKeepsOutsideContents(t *testing.T) { + fs, outside := newLinkedRoot(t, true) + + _ = fs.RemoveAll("link/sub") + require.DirExists(t, filepath.Join(outside, "sub")) + + _ = fs.RemoveAll("link") + require.FileExists(t, filepath.Join(outside, "file.txt")) + require.DirExists(t, filepath.Join(outside, "sub")) +} diff --git a/internal/ufs/mkdir_unix.go b/internal/ufs/mkdir_unix.go index 79afb2488..505fa7e0f 100644 --- a/internal/ufs/mkdir_unix.go +++ b/internal/ufs/mkdir_unix.go @@ -10,6 +10,8 @@ package ufs +import "errors" + // mkdirAll is a recursive Mkdir implementation that properly handles symlinks. // // It returns the directories it created, ordered from shallowest to deepest, so @@ -21,8 +23,12 @@ func (fs *UnixFS) mkdirAll(name string, mode FileMode) ([]string, error) { if err == nil { if dir.Mode()&ModeSymlink != 0 { // If the final path is a symlink, resolve its target and use that - // to check instead. + // to check instead. A link that leads outside the filesystem is + // not a directory that can be used. dir, err = fs.Stat(name) + if errors.Is(err, ErrBadPathResolution) { + return nil, &PathError{Op: "mkdir", Path: name, Err: ErrNotDirectory} + } if err != nil { return nil, err } diff --git a/internal/ufs/removeall_unix.go b/internal/ufs/removeall_unix.go index d756021af..fe551a6b3 100644 --- a/internal/ufs/removeall_unix.go +++ b/internal/ufs/removeall_unix.go @@ -177,7 +177,7 @@ func removeContentsFrom(fs unixFS, parent File, base string) error { } } - return nil + return recurseErr } func removeAllFrom(fs unixFS, parent File, base string) error { @@ -231,7 +231,7 @@ func removeAllFrom(fs unixFS, parent File, base string) error { return recurseErr } - return ensurePathError(err, "unlinkat", base) + return ensurePathError(unlinkErr, "unlinkat", base) } // openFdAt opens path relative to the directory in fd. @@ -243,7 +243,8 @@ func openFdAt(dirfd int, name string) (File, error) { var fd int for { var err error - fd, err = unix.Openat(dirfd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW, 0) + // This is only used to open directories. + fd, err = unix.Openat(dirfd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_DIRECTORY, 0) if err == nil { break } diff --git a/internal/ufs/removeall_unix_test.go b/internal/ufs/removeall_unix_test.go new file mode 100644 index 000000000..995fee336 --- /dev/null +++ b/internal/ufs/removeall_unix_test.go @@ -0,0 +1,53 @@ +//go:build unix + +package ufs + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +// failingUnlinkFS fails to remove any entry with the given name. +type failingUnlinkFS struct { + *UnixFS + name string +} + +func (fs failingUnlinkFS) unlinkat(dirfd int, name string, flags int) error { + if name == fs.name { + return unix.EBUSY + } + return fs.UnixFS.unlinkat(dirfd, name, flags) +} + +func TestRemoveReportsEntriesThatCouldNotBeRemoved(t *testing.T) { + root := t.TempDir() + for _, p := range []string{"dir/keep", "dir/remove"} { + if err := os.MkdirAll(filepath.Join(root, filepath.Dir(p)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, p), nil, 0o600); err != nil { + t.Fatal(err) + } + } + unixFS, err := NewUnixFS(root, false) + if err != nil { + t.Fatal(err) + } + defer unixFS.Close() + fs := failingUnlinkFS{UnixFS: unixFS, name: "keep"} + + if err := removeContents(fs, "dir"); !errors.Is(err, unix.EBUSY) { + t.Fatalf("expected removing the contents to report the entry that could not be removed, got %v", err) + } + if err := removeAll(fs, "dir"); !errors.Is(err, unix.EBUSY) { + t.Fatalf("expected removing the directory to report the entry that could not be removed, got %v", err) + } + if _, err := os.Stat(filepath.Join(root, "dir/remove")); !os.IsNotExist(err) { + t.Fatal("expected the other entry to be removed") + } +} diff --git a/internal/ufs/walk_unix.go b/internal/ufs/walk_unix.go index 065afc229..e10c7bcff 100644 --- a/internal/ufs/walk_unix.go +++ b/internal/ufs/walk_unix.go @@ -48,6 +48,12 @@ func (fs *UnixFS) walkDir(b []byte, parentfd int, name, relative string, d DirEn defer unix.Close(dirfd) } if err != nil { + // Report the error to the callback, the same as a failed read below, so + // that it can decide whether to continue. + err = walkDirFn(parentfd, name, relative, d, err) + if err == SkipDir && d.IsDir() { + err = nil + } return err } @@ -139,10 +145,10 @@ func nameFromDirent(de *unix.Dirent) (name []byte) { // SliceHeader was mainly deprecated due to it being misused for avoiding // allocations when converting a byte slice to a string, ref; // https://go.dev/issue/53003 - sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) //nolint:gosec,staticcheck // audited, deprecation tracked by the TODO above sh.Cap = ml sh.Len = ml - sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) //nolint:gosec // audited, see above if index := bytes.IndexByte(name, 0); index >= 0 { // Found NULL byte; set slice's cap and len accordingly. @@ -237,6 +243,7 @@ func (fs *UnixFS) readDir(fd int, name, relative string, b []byte) ([]DirEntry, } // "Go is like C, except that you just put `unsafe` all over the place". + //nolint:gosec // bounded by the size of sde copy((*[unsafe.Sizeof(unix.Dirent{})]byte)(unsafe.Pointer(&sde))[:], workBuffer) workBuffer = workBuffer[sde.Reclen:] // advance buffer for next iteration through loop @@ -304,11 +311,3 @@ func (de dirent) Open() (File, error) { return de.fs.OpenFileat(de.dirfd, de.name, O_RDONLY, 0) // return de.fs.OpenFile(de.path, O_RDONLY, 0) } - -// reset releases memory held by entry err and name, and resets mode type to 0. -func (de *dirent) reset() { - de.name = "" - de.path = "" - de.modeType = 0 - de.dirfd = 0 -} diff --git a/internal/ufs/walk_unix_test.go b/internal/ufs/walk_unix_test.go new file mode 100644 index 000000000..5c17e1f37 --- /dev/null +++ b/internal/ufs/walk_unix_test.go @@ -0,0 +1,69 @@ +//go:build unix + +package ufs_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// A directory that is removed after it has been listed, but before the walk +// opens it, is reported to the callback rather than ending the walk. +func TestWalkDiratReportsDirectoriesRemovedDuringWalk(t *testing.T) { + t.Parallel() + + fs, err := newTestUnixFS() + if err != nil { + t.Fatal(err) + } + defer fs.Cleanup() + + for _, dir := range []string{"a", "b", "c"} { + if err := os.MkdirAll(filepath.Join(fs.Root, dir, "child"), 0o755); err != nil { + t.Fatal(err) + } + } + + dirfd, name, closeFd, err := fs.SafePath("/") + defer closeFd() + if err != nil { + t.Fatal(err) + } + + var reported []string + var visited []string + err = fs.WalkDirat(dirfd, name, func(_ int, _, relative string, d ufs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, ufs.ErrNotExist) { + reported = append(reported, relative) + return nil + } + return err + } + visited = append(visited, relative) + // Remove the directory after it has been listed and before it is opened. + if relative == "b" { + return os.RemoveAll(filepath.Join(fs.Root, "b")) + } + return nil + }) + if err != nil { + t.Fatalf("expected the walk to continue past a removed directory, got %v", err) + } + if len(reported) != 1 || reported[0] != "b" { + t.Fatalf("expected the removed directory to be reported, got %v", reported) + } + for _, want := range []string{"a/child", "c/child"} { + found := false + for _, v := range visited { + found = found || v == want + } + if !found { + t.Fatalf("expected %s to be visited, visited %v", want, visited) + } + } +} diff --git a/parser/helpers.go b/parser/helpers.go index a6aed945d..01d21a4d8 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -1,7 +1,6 @@ package parser import ( - "bytes" "regexp" "strconv" "strings" @@ -188,8 +187,7 @@ func (cfr *ConfigurationFileReplacement) SetAtPathway(c *gabs.Container, path st // Check if we are replacing instead of overwriting. if strings.HasPrefix(cfr.IfValue, "regex:") { // Doing a regex replacement requires an existing value. - // TODO: Do we try passing an empty string to the regex? - if c.ExistsP(path) { + if !c.ExistsP(path) { return gabs.ErrNotFound } @@ -200,20 +198,30 @@ func (cfr *ConfigurationFileReplacement) SetAtPathway(c *gabs.Container, path st return nil } - v := strings.Trim(c.Path(path).String(), "\"") - if r.Match([]byte(v)) { + v := valueAtPath(c, path) + if r.MatchString(v) { return setValueAtPath(c, path, r.ReplaceAllString(v, value)) } return nil } - if c.ExistsP(path) && !bytes.Equal(c.Bytes(), []byte(cfr.IfValue)) { + if c.ExistsP(path) && valueAtPath(c, path) != cfr.IfValue { return nil } return setValueAtPath(c, path, cfr.getKeyValue(value)) } +// Returns the value at the path as plain text: strings without their JSON quotes, and +// numbers and booleans as written, so they compare against an if_value directly. +func valueAtPath(c *gabs.Container, path string) string { + if s, ok := c.Path(path).Data().(string); ok { + return s + } + + return c.Path(path).String() +} + // Looks up a configuration value on the Daemon given a dot-notated syntax. func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplacement) (string, error) { // If this is not something that we can do a regex lookup on then just continue @@ -247,7 +255,7 @@ func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplac // If there is no key, keep the original value intact, that way it is obvious there // is a replace issue at play. - return string(match), nil + return cfr.ReplaceWith.String(), nil } // Only substitute scalar values, not whole objects or arrays. diff --git a/parser/helpers_test.go b/parser/helpers_test.go new file mode 100644 index 000000000..5129dafa7 --- /dev/null +++ b/parser/helpers_test.go @@ -0,0 +1,134 @@ +package parser + +import ( + "encoding/json" + "testing" + + "github.com/Jeffail/gabs/v2" + "github.com/buger/jsonparser" +) + +func replacement(t *testing.T, raw string) ConfigurationFileReplacement { + t.Helper() + + var cfr ConfigurationFileReplacement + if err := json.Unmarshal([]byte(raw), &cfr); err != nil { + t.Fatalf("unmarshal %s: %v", raw, err) + } + + return cfr +} + +func TestSetAtPathway(t *testing.T) { + cases := []struct { + name string + document string + replacement string + value string + path string + want string + }{ + { + name: "no if_value replaces", + document: `{"server":{"port":25565}}`, + replacement: `{"match":"server.port","replace_with":"25566"}`, + value: "25566", + path: "server.port", + want: "25566", + }, + { + name: "if_value matches the key's value", + document: `{"listeners":{"host":"0.0.0.0:25577","motd":"hi"}}`, + replacement: `{"match":"listeners.host","if_value":"0.0.0.0:25577","replace_with":"0.0.0.0:30000"}`, + value: "0.0.0.0:30000", + path: "listeners.host", + want: `"0.0.0.0:30000"`, + }, + { + name: "if_value does not match", + document: `{"listeners":{"host":"127.0.0.1:25577"}}`, + replacement: `{"match":"listeners.host","if_value":"0.0.0.0:25577","replace_with":"0.0.0.0:30000"}`, + value: "0.0.0.0:30000", + path: "listeners.host", + want: `"127.0.0.1:25577"`, + }, + { + name: "numeric if_value matches a numeric value", + document: `{"server":{"port":25565}}`, + replacement: `{"match":"server.port","if_value":25565,"replace_with":"30000"}`, + value: "30000", + path: "server.port", + want: "30000", + }, + { + name: "regex if_value rewrites an existing value", + document: `{"servers":{"lobby":{"address":"localhost:25565"}}}`, + replacement: `{"match":"servers.lobby.address","if_value":"regex:^(127\\.0\\.0\\.1|localhost)(:\\d{1,5})?$","replace_with":"10.0.0.5$2"}`, + value: "10.0.0.5$2", + path: "servers.lobby.address", + want: `"10.0.0.5:25565"`, + }, + { + name: "regex if_value leaves a non-matching value", + document: `{"servers":{"lobby":{"address":"play.example.com:25565"}}}`, + replacement: `{"match":"servers.lobby.address","if_value":"regex:^(127\\.0\\.0\\.1|localhost)(:\\d{1,5})?$","replace_with":"10.0.0.5$2"}`, + value: "10.0.0.5$2", + path: "servers.lobby.address", + want: `"play.example.com:25565"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, err := gabs.ParseJSON([]byte(tc.document)) + if err != nil { + t.Fatal(err) + } + + cfr := replacement(t, tc.replacement) + if err := cfr.SetAtPathway(c, cfr.Match, tc.value); err != nil { + t.Fatalf("SetAtPathway: %v", err) + } + if got := c.Path(tc.path).String(); got != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + }) + } +} + +func TestSetAtPathwayRegexOnMissingKey(t *testing.T) { + c, _ := gabs.ParseJSON([]byte(`{"servers":{}}`)) + cfr := replacement(t, `{"match":"servers.lobby.address","if_value":"regex:^localhost","replace_with":"10.0.0.5"}`) + + if err := cfr.SetAtPathway(c, cfr.Match, "10.0.0.5"); err != gabs.ErrNotFound { + t.Fatalf("expected gabs.ErrNotFound for a missing key, got %v", err) + } + if c.ExistsP("servers.lobby.address") { + t.Fatal("a regex replacement must not create a missing key") + } +} + +func TestReplacementAcceptsNumericKeys(t *testing.T) { + cfr := replacement(t, `{"match":25565,"if_value":true,"replace_with":"{{server.build.default.port}}"}`) + + if cfr.Match != "25565" { + t.Fatalf("match: got %q", cfr.Match) + } + if cfr.IfValue != "true" { + t.Fatalf("if_value: got %q", cfr.IfValue) + } + if cfr.ReplaceWith.Type() != jsonparser.String { + t.Fatalf("replace_with type: got %v", cfr.ReplaceWith.Type()) + } +} + +func TestConfigurationFileKeepsReplacementsWithNumericMatch(t *testing.T) { + var f ConfigurationFile + raw := `{"file":"server.properties","parser":"properties","replace":[{"match":25565,"replace_with":"a"},{"match":"motd","replace_with":"b"}]}` + if err := json.Unmarshal([]byte(raw), &f); err != nil { + t.Fatal(err) + } + if len(f.Replace) != 2 { + t.Fatalf("expected both replacements to survive decoding, got %d", len(f.Replace)) + } +} diff --git a/parser/parser.go b/parser/parser.go index 58a0a4204..04a1df6fc 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -37,6 +37,34 @@ const maxTextScanTokenSize = 64 * 1024 * 1024 // maxConfigFileSize caps how large a configuration file we'll attempt to parse. const maxConfigFileSize = 64 * 1024 * 1024 +// maxXMLDepth is the deepest element nesting of an XML configuration file that +// is rewritten. Real configuration files are far shallower than this. +const maxXMLDepth = 512 + +// xmlDepth returns the element nesting depth of the tree rooted at root. +func xmlDepth(root *etree.Element) int { + if root == nil { + return 0 + } + type frame struct { + e *etree.Element + d int + } + stack := []frame{{root, 1}} + depth := 0 + for len(stack) > 0 { + f := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if f.d > depth { + depth = f.d + } + for _, c := range f.e.ChildElements() { + stack = append(stack, frame{c, f.d + 1}) + } + } + return depth +} + type ReplaceValue struct { value []byte valueType jsonparser.ValueType @@ -128,6 +156,11 @@ func (f *ConfigurationFile) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &m); err != nil { return err } + for _, key := range []string{"file", "parser", "replace"} { + if m[key] == nil { + return errors.Errorf("parser: configuration file is missing the %q field", key) + } + } if err := json.Unmarshal(*m["file"], &f.FileName); err != nil { return err @@ -148,6 +181,26 @@ func (f *ConfigurationFile) UnmarshalJSON(data []byte) error { // ConfigurationFileReplacement defines a single find/replace instance for a // given server configuration file. +// Reads a key as text. The Panel encodes a numeric key such as "25565" as a JSON number, +// so numbers and booleans are accepted alongside strings. +func scalarString(data []byte, key string) (string, error) { + value, dataType, _, err := jsonparser.Get(data, key) + if err != nil { + return "", err + } + + switch dataType { + case jsonparser.String: + return jsonparser.ParseString(value) + case jsonparser.Number, jsonparser.Boolean: + return string(value), nil + case jsonparser.Null: + return "", nil + default: + return "", errors.New("parser: " + key + " must be a string, number, or boolean") + } +} + type ConfigurationFileReplacement struct { Match string `json:"match"` IfValue string `json:"if_value"` @@ -157,14 +210,14 @@ type ConfigurationFileReplacement struct { // UnmarshalJSON handles unmarshaling the JSON representation into a struct that // provides more useful data to this functionality. func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error { - m, err := jsonparser.GetString(data, "match") + m, err := scalarString(data, "match") if err != nil { return err } cfr.Match = m - iv, err := jsonparser.GetString(data, "if_value") + iv, err := scalarString(data, "if_value") // We only check keypath here since match & replace_with should be present on all of // them, however if_value is optional. if err != nil && err != jsonparser.KeyPathNotFoundError { @@ -256,6 +309,11 @@ func (f *ConfigurationFile) parseXmlFile(file ufs.File) error { if _, err := doc.ReadFrom(io.LimitReader(file, maxConfigFileSize)); err != nil { return err } + // The server still boots with the file as it is, the same as for files over + // the size limit. + if depth := xmlDepth(doc.Root()); depth > maxXMLDepth { + return errors.Errorf("parser: refusing to parse configuration file %q: element nesting depth %d exceeds limit of %d", file.Name(), depth, maxXMLDepth) + } // If there is no root we should create a basic start to the file. This isn't required though, // and if it doesn't work correctly I'll just remove the code. @@ -276,7 +334,7 @@ func (f *ConfigurationFile) parseXmlFile(file ufs.File) error { doc.SetRoot(doc.CreateElement(parts[0])) } - path := "./" + strings.Replace(replacement.Match, ".", "/", -1) + path := "./" + strings.ReplaceAll(replacement.Match, ".", "/") // If we're not doing a wildcard replacement go ahead and create the // missing element if we cannot find it yet. @@ -312,21 +370,14 @@ func (f *ConfigurationFile) parseXmlFile(file ufs.File) error { } } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - // Ensure the XML is indented properly. doc.Indent(2) - // Write the XML to the file. - if _, err := doc.WriteTo(file); err != nil { + var buf bytes.Buffer + if _, err := doc.WriteTo(&buf); err != nil { return err } - return nil + return rewrite(file, buf.Bytes()) } // Parses an ini file. @@ -395,17 +446,11 @@ func (f *ConfigurationFile) parseIniFile(file ufs.File) error { } } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - if _, err := cfg.WriteTo(file); err != nil { + var buf bytes.Buffer + if _, err := cfg.WriteTo(&buf); err != nil { return err } - return nil + return rewrite(file, buf.Bytes()) } // Parses a json file updating any matching key/value pairs. If a match is not found, the @@ -422,18 +467,7 @@ func (f *ConfigurationFile) parseJsonFile(file ufs.File) error { return err } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - // Write the data to the file. - if _, err := io.Copy(file, bytes.NewReader(data.BytesIndent("", " "))); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } - return nil + return rewrite(file, data.BytesIndent("", " ")) } // Parses a yaml file and updates any matching key/value pairs before persisting @@ -470,18 +504,7 @@ func (f *ConfigurationFile) parseYamlFile(file ufs.File) error { return err } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - // Write the data to the file. - if _, err := io.Copy(file, bytes.NewReader(marshaled)); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } - return nil + return rewrite(file, marshaled) } // Parses a text file using basic find and replace. This is a highly inefficient method of @@ -508,23 +531,16 @@ func (f *ConfigurationFile) parseTextFile(file ufs.File) error { b.Write(line) } b.WriteByte('\n') + // Apply the same limit to the result. + if b.Len() > maxConfigFileSize { + return errors.Errorf("parser: refusing to write configuration file %q: result exceeds limit of %d bytes", file.Name(), maxConfigFileSize) + } } if err := s.Err(); err != nil { return errors.Wrap(err, "parser: failed to scan text file for configuration update") } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - // Write the data to the file. - if _, err := io.Copy(file, b); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } - return nil + return rewrite(file, b.Bytes()) } // parsePropertiesFile parses a properties file and updates the values within it @@ -613,16 +629,35 @@ func (f *ConfigurationFile) parsePropertiesFile(file ufs.File) error { s.WriteString(key + "=" + strings.Trim(strconv.QuoteToASCII(value), "\"") + "\n") } + return rewrite(file, s.Bytes()) +} + +// sizeChecker is implemented by files whose size counts against a disk limit. +type sizeChecker interface { + // CheckSize returns an error if the file cannot grow to size bytes. + CheckSize(size int64) error +} + +// rewrite replaces the contents of file with data. The file is left unchanged if +// data is too large, or would not fit in the disk space the file may use. +func rewrite(file ufs.File, data []byte) error { + if len(data) > maxConfigFileSize { + return errors.Errorf("parser: refusing to write configuration file %q: result exceeds limit of %d bytes", file.Name(), maxConfigFileSize) + } + if c, ok := file.(sizeChecker); ok { + if err := c.CheckSize(int64(len(data))); err != nil { + return err + } + } + if _, err := file.Seek(0, io.SeekStart); err != nil { return err } if err := file.Truncate(0); err != nil { return err } - - // Write the data to the file. - if _, err := io.Copy(file, s); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") + if _, err := file.Write(data); err != nil { + return errors.Wrap(err, "parser: failed to write configuration file to disk") } return nil } diff --git a/parser/parser_fields_test.go b/parser/parser_fields_test.go new file mode 100644 index 000000000..4eac0bf2e --- /dev/null +++ b/parser/parser_fields_test.go @@ -0,0 +1,57 @@ +package parser + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Control: a present key is substituted into the placeholder. +func TestPresentConfigurationKeySubstitutes(t *testing.T) { + f := ConfigurationFile{configuration: []byte(`{"docker":{"interface":"eth0"}}`)} + cfr := replacement(t, `{"match":"bind-address","replace_with":"{{config.docker.interface}}"}`) + + got, err := f.LookupConfigurationValue(cfr) + require.NoError(t, err) + assert.Equal(t, "eth0", got) +} + +// A placeholder for a key that does not exist is left in place. +func TestMissingConfigurationKeyKeepsPlaceholder(t *testing.T) { + f := ConfigurationFile{configuration: []byte(`{"docker":{"interface":"eth0"}}`)} + cfr := replacement(t, `{"match":"bind-address","replace_with":"{{config.docker.missing}}"}`) + + got, err := f.LookupConfigurationValue(cfr) + require.NoError(t, err) + assert.Equal(t, "{{config.docker.missing}}", got, "a missing key should leave the placeholder in place") +} + +// A configuration file entry that is missing a field is refused when it is decoded. +func TestConfigurationFileWithoutParserKeyReturnsError(t *testing.T) { + var f ConfigurationFile + var err error + require.NotPanics(t, func() { + err = json.Unmarshal([]byte(`{"file":"server.properties","replace":[]}`), &f) + }) + require.Error(t, err) + + for _, raw := range []string{`{"parser":"file","replace":[]}`, `{"file":"a","parser":"file"}`} { + require.NotPanics(t, func() { + err = json.Unmarshal([]byte(raw), &f) + }) + require.Error(t, err, raw) + } +} + +// An invalid escape in replace_with is refused while the document is validated, +// before any replacement value is built. +func TestInvalidReplacementEscapeIsRejectedBeforeDecoding(t *testing.T) { + var f ConfigurationFile + var err error + require.NotPanics(t, func() { + err = json.Unmarshal([]byte(`{"file":"a","parser":"file","replace":[{"match":"k","replace_with":"\z"}]}`), &f) + }) + require.Error(t, err) +} diff --git a/parser/parser_xml_test.go b/parser/parser_xml_test.go new file mode 100644 index 000000000..67813eb7d --- /dev/null +++ b/parser/parser_xml_test.go @@ -0,0 +1,36 @@ +package parser + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// An XML file nested deeper than maxXMLDepth is left as it is. +func TestNestedXMLOverDepthLimitIsNotRewritten(t *testing.T) { + const depth = 10000 + content := strings.Repeat("", depth) + strings.Repeat("", depth) + path := filepath.Join(t.TempDir(), "config.xml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + var cf ConfigurationFile + require.NoError(t, json.Unmarshal([]byte(`{"file":"config.xml","parser":"xml","replace":[{"match":"a","replace_with":"value"}]}`), &cf)) + cf.configuration = []byte(`{}`) + + f, err := os.OpenFile(path, os.O_RDWR, 0) + require.NoError(t, err) + defer f.Close() + + start := time.Now() + require.Error(t, cf.parseXmlFile(f)) + require.Less(t, time.Since(start), 5*time.Second) + + b, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, content, string(b)) +} diff --git a/remote/types.go b/remote/types.go index d9022480a..4e9efb55a 100644 --- a/remote/types.go +++ b/remote/types.go @@ -96,13 +96,15 @@ type OutputLineMatcher struct { // `regex:` which indicates we want to match against the regex expression. raw []byte reg *regexp.Regexp + // literal is the text matched when there is no regex. + literal []byte } // Matches determines if the provided byte string matches the given regex or // raw string provided to the matcher. func (olm *OutputLineMatcher) Matches(s []byte) bool { if olm.reg == nil { - return bytes.Contains(s, olm.raw) + return bytes.Contains(s, olm.literal) } return olm.reg.Match(s) } @@ -121,10 +123,15 @@ func (olm *OutputLineMatcher) UnmarshalJSON(data []byte) error { } olm.raw = []byte(r) + olm.literal = olm.raw if bytes.HasPrefix(olm.raw, []byte("regex:")) && len(olm.raw) > 6 { - r, err := regexp.Compile(strings.TrimPrefix(string(olm.raw), "regex:")) + pattern := strings.TrimPrefix(string(olm.raw), "regex:") + r, err := regexp.Compile(pattern) if err != nil { - log.WithField("error", err).WithField("raw", string(olm.raw)).Warn("failed to compile output line marked as being regex") + // Match the expression as plain text instead, rather than never + // matching, which would leave the server starting forever. + log.WithField("error", err).WithField("raw", string(olm.raw)).Warn("failed to compile output line marked as being regex, matching it as text") + olm.literal = []byte(pattern) } olm.reg = r } diff --git a/remote/types_test.go b/remote/types_test.go new file mode 100644 index 000000000..b3806ff41 --- /dev/null +++ b/remote/types_test.go @@ -0,0 +1,31 @@ +package remote + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOutputLineMatcher(t *testing.T) { + for _, tt := range []struct { + raw string + line string + match bool + }{ + {raw: "Done", line: "[12:00] Done (1.2s)!", match: true}, + {raw: "Done", line: "Loading", match: false}, + {raw: `regex:^\[\d+:\d+\] Done`, line: "[12:00] Done (1.2s)!", match: true}, + {raw: `regex:^Done`, line: "[12:00] Done", match: false}, + // An expression that does not compile is matched as plain text. + {raw: "regex:Done (", line: "[12:00] Done (1.2s)!", match: true}, + {raw: "regex:Done (", line: "[12:00] Loading", match: false}, + } { + var m OutputLineMatcher + b, err := json.Marshal(tt.raw) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(b, &m)) + require.Equal(t, tt.match, m.Matches([]byte(tt.line)), "%q against %q", tt.raw, tt.line) + require.Equal(t, tt.raw, m.String()) + } +} diff --git a/router/content_type_test.go b/router/content_type_test.go index d68649f19..e87b21c40 100644 --- a/router/content_type_test.go +++ b/router/content_type_test.go @@ -21,23 +21,23 @@ func TestIsValidBackupContentType(t *testing.T) { // TAR formats {"TAR x-tar", "application/x-tar", true}, {"TAR tar", "application/tar", true}, - + // Generic formats {"Octet stream", "application/octet-stream", true}, {"Binary octet", "binary/octet-stream", true}, - + // Backup specific {"Compressed tar", "application/x-compressed-tar", true}, {"TGZ", "application/x-tgz", true}, - + // Case insensitive {"Uppercase GZIP", "APPLICATION/X-GZIP", true}, {"Mixed case", "Application/Octet-Stream", true}, - + // With charset parameters {"GZIP with charset", "application/x-gzip; charset=binary", true}, {"Octet with boundary", "application/octet-stream; boundary=something", true}, - + // Invalid types {"Plain text", "text/plain", false}, {"HTML", "text/html", false}, @@ -56,4 +56,4 @@ func TestIsValidBackupContentType(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go index a6fcaf63f..7a6c0cec5 100644 --- a/router/downloader/downloader.go +++ b/router/downloader/downloader.go @@ -8,16 +8,20 @@ import ( "mime" "net" "net/http" + "net/netip" "net/url" "path/filepath" "strings" "sync" + "syscall" "time" "emperror.dev/errors" "github.com/apex/log" "github.com/google/uuid" + "github.com/Rene-Roscher/wings/config" + wnet "github.com/Rene-Roscher/wings/internal/network" "github.com/Rene-Roscher/wings/server" ) @@ -27,33 +31,35 @@ func init() { dialer := &net.Dialer{ LocalAddr: nil, Timeout: time.Second * 30, + // Check every address immediately before connecting to it, since the + // destination is resolved again when dialing. + ControlContext: checkConnection, } trnspt := http.DefaultTransport.(*http.Transport).Clone() + // Always connect directly, so that every destination is checked. + trnspt.Proxy = nil + // Store the file exactly as it is served. Asking for a compressed response + // would also hide the length of the file, which is needed to write it. + trnspt.DisableCompression = true trnspt.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - c, err := dialer.DialContext(ctx, network, addr) + host, _, err := net.SplitHostPort(addr) if err != nil { return nil, errors.WithStack(err) } - - ipStr, _, err := net.SplitHostPort(c.RemoteAddr().String()) - if err != nil { - return c, errors.WithStack(err) - } - ip := net.ParseIP(ipStr) - if ip == nil { - return c, errors.WithStack(ErrInvalidIPAddress) + // Refuse the destination if any of its addresses are internal, or if it + // cannot be resolved. + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil || len(ips) == 0 { + log.WithField("host", host).WithField("error", err).Debug("downloader: failed to resolve remote file host") + return nil, errors.WithStack(ErrInvalidIPAddress) } - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() { - return c, errors.WithStack(ErrInternalResolution) - } - for _, block := range internalRanges { - if !block.Contains(ip) { - continue + for _, ip := range ips { + if isBlocked(host, ip) { + return nil, errors.WithStack(ErrInternalResolution) } - return c, errors.WithStack(ErrInternalResolution) } - return c, nil + return dialer.DialContext(context.WithValue(ctx, dialHostKey{}, host), network, addr) } client = &http.Client{ @@ -74,6 +80,34 @@ func init() { } } +// dialHostKey is the context key for the host name a connection is being made +// to, which is checked against the allowlist when the connection is made. +type dialHostKey struct{} + +// checkConnection is called with the address a connection is about to be made +// to, and refuses to connect to any internal address. +func checkConnection(ctx context.Context, _ string, address string, _ syscall.RawConn) error { + ipStr, _, err := net.SplitHostPort(address) + if err != nil { + return errors.WithStack(err) + } + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return errors.WithStack(ErrInternalResolution) + } + host, _ := ctx.Value(dialHostKey{}).(string) + if isBlocked(host, ip) { + return errors.WithStack(ErrInternalResolution) + } + return nil +} + +// isBlocked reports whether connecting to the address, resolved from the host, +// is refused because it is internal and not allowed by the operator. +func isBlocked(host string, ip netip.Addr) bool { + return wnet.IsInternal(ip) && !wnet.IsAllowed(config.Get().Api.RemoteDownloadAllowlist, host, ip) +} + var instance = &Downloader{ // Tracks all the active downloads. downloadCache: make(map[string]*Download), @@ -83,18 +117,6 @@ var instance = &Downloader{ serverCache: make(map[string][]string), } -// Internal IP ranges that should be blocked if the resource requested resolves within. -var internalRanges = []*net.IPNet{ - mustParseCIDR("127.0.0.1/8"), - mustParseCIDR("10.0.0.0/8"), - mustParseCIDR("172.16.0.0/12"), - mustParseCIDR("192.168.0.0/16"), - mustParseCIDR("169.254.0.0/16"), - mustParseCIDR("::1/128"), - mustParseCIDR("fe80::/10"), - mustParseCIDR("fc00::/7"), -} - const ( ErrInternalResolution = errors.Sentinel("downloader: destination resolves to internal network location") ErrInvalidIPAddress = errors.Sentinel("downloader: invalid IP address") @@ -192,8 +214,7 @@ func ByID(dlid string) *Download { return instance.find(dlid) } -//goland:noinspection GoVetCopyLock -func (dl Download) MarshalJSON() ([]byte, error) { +func (dl *Download) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Identifier string Progress float64 @@ -287,7 +308,10 @@ func (dl *Download) Execute() error { // Execute request res, err = client.Do(req) if err != nil { - return ErrDownloadFailed + if IsDownloadError(err) { + return err + } + return errors.WithStack(fmt.Errorf("%w: %w", ErrDownloadFailed, err)) } // Check if this is a redirect (3xx status codes) @@ -377,9 +401,16 @@ func (dl *Download) Execute() error { return errors.WrapIf(err, "downloader: invalid \"Content-Disposition\" header") } + // Only use the name of the file, so that it is saved in the + // directory that was asked for. if v, ok := params["filename"]; ok { - // SECURITY FIX: Sanitize Content-Disposition filename (Attack Vector #1) - unsafeFilename = v + // SECURITY FIX: Sanitize Content-Disposition filename (Attack Vector #1). + // Only use the name of the file, so that it is saved in the + // directory that was asked for (upstream v1.13.4); the result is + // still validated by sanitizeFilename below. + if name := filepath.Base(filepath.Clean("/" + v)); name != "/" && name != "." { + unsafeFilename = name + } } } } @@ -444,17 +475,58 @@ func (dl *Download) Execute() error { }() p := dl.Path() + if err := dl.server.Filesystem().IsIgnored(p); err != nil { + return err + } dl.server.Log().WithField("path", p).Debug("writing remote file to disk") // Write the file while tracking the progress, Write will check that the // size of the file won't exceed the disk limit. - r := io.TeeReader(res.Body, dl.counter(res.ContentLength)) - if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil { + // A response without a known length (chunked) is never checked against + // maxDownloadSize above, so cap the body itself. + body := &maxSizeReader{r: res.Body, remaining: maxDownloadSize} + r := io.TeeReader(body, dl.counter(res.ContentLength)) + if hasKnownSize { + err = dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644) + } else { + // Write copies nothing for an unknown (-1) size; WriteFrom counts each + // write against the disk limit and removes the file if it fails. + _, err = dl.server.Filesystem().WriteFrom(p, r) + } + if err != nil { + if body.exceeded { + return errors.Wrap(ErrFileTooLarge, fmt.Sprintf("file exceeds maximum %d bytes", maxDownloadSize)) + } return errors.WrapIf(err, "downloader: failed to write file to server directory") } return nil } +// maxSizeReader returns ErrFileTooLarge once more than remaining bytes have +// been read, instead of silently truncating like io.LimitReader. +type maxSizeReader struct { + r io.Reader + remaining int64 + exceeded bool +} + +func (m *maxSizeReader) Read(p []byte) (int, error) { + if m.remaining < 0 { + m.exceeded = true + return 0, ErrFileTooLarge + } + if int64(len(p)) > m.remaining+1 { + p = p[:m.remaining+1] + } + n, err := m.r.Read(p) + m.remaining -= int64(n) + if m.remaining < 0 { + m.exceeded = true + return n, ErrFileTooLarge + } + return n, err +} + // Cancel cancels a running download and frees up the associated resources. If a file is being // written a partial file will remain present on the disk. func (dl *Download) Cancel() { @@ -521,7 +593,7 @@ func (dl *Download) sendProgressEvent(percentage int, bytesWritten, bytesTotal i // Throttle: Only send events every 250ms (like backup progress) // ALWAYS send: 0%, 100%, or status change const throttleNanos = 250_000_000 // 250ms - shouldSend := (now - dl.lastEventTime) >= throttleNanos || + shouldSend := (now-dl.lastEventTime) >= throttleNanos || percentage != dl.lastPercentage || status != "downloading" @@ -600,14 +672,6 @@ func (d *Downloader) remove(dlID string) { } } -func mustParseCIDR(ip string) *net.IPNet { - _, block, err := net.ParseCIDR(ip) - if err != nil { - panic(fmt.Errorf("downloader: failed to parse CIDR: %s", err)) - } - return block -} - func IsDownloadError(err error) bool { return errors.Is(err, ErrDownloadFailed) || errors.Is(err, ErrInvalidIPAddress) || errors.Is(err, ErrInternalResolution) } diff --git a/router/downloader/downloader_pull_test.go b/router/downloader/downloader_pull_test.go new file mode 100644 index 000000000..408852819 --- /dev/null +++ b/router/downloader/downloader_pull_test.go @@ -0,0 +1,132 @@ +package downloader + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/server" +) + +const pullTestServerUuid = "ffffffff-ffff-4fff-8fff-ffffffffffff" + +// newPullTestServer returns a server whose files are stored in a temporary data +// directory and owned by the user running the tests, so that they pass without +// root. Loopback is allowed so that the tests can use a local origin. +func newPullTestServer(t *testing.T) *server.Server { + t.Helper() + previous := config.Get() + t.Cleanup(func() { config.Set(previous) }) + next := *previous + next.System.Data = t.TempDir() + next.System.User.Uid = os.Getuid() + next.System.User.Gid = os.Getgid() + next.Api.RemoteDownloadAllowlist = []string{"127.0.0.1"} + config.Set(&next) + + settings, err := json.Marshal(map[string]string{"uuid": pullTestServerUuid}) + require.NoError(t, err) + s, err := server.NewEmptyManager(nil).InitServer(pullTestServerUuid, remote.ServerConfigurationResponse{ + Settings: settings, + ProcessConfiguration: &remote.ProcessConfiguration{}, + }) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(s.Filesystem().Path(), 0o755)) + return s +} + +// A file from a server that compresses responses for clients that accept gzip +// can be pulled. +func TestDownloadAcceptsGzipEncodedResponses(t *testing.T) { + s := newPullTestServer(t) + + payload := []byte(strings.Repeat("pull test payload\n", 4096)) + var compressed bytes.Buffer + zw := gzip.NewWriter(&compressed) + _, err := zw.Write(payload) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := payload + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + w.Header().Set("Content-Encoding", "gzip") + body = compressed.Bytes() + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + _, _ = w.Write(body) + })) + t.Cleanup(origin.Close) + + u, err := url.Parse(origin.URL + "/payload.bin") + require.NoError(t, err) + require.NoError(t, New(s, DownloadRequest{Directory: "/", URL: u, FileName: "payload.bin"}).Execute()) + + got, err := os.ReadFile(filepath.Join(s.Filesystem().Path(), "payload.bin")) + require.NoError(t, err) + require.Equal(t, payload, got) +} + +// Only the name of a file from Content-Disposition is used, so the file is saved +// in the directory that was asked for. +func TestDownloadFilenameFromHeaderIsSavedInRequestedDirectory(t *testing.T) { + tests := []struct { + directory string + want string + }{ + {directory: "/chosen/dir", want: "chosen/dir/named.txt"}, + {directory: "chosen/dir", want: "chosen/dir/named.txt"}, + {directory: "", want: "named.txt"}, + } + for _, tt := range tests { + t.Run(tt.directory, func(t *testing.T) { + s := newPullTestServer(t) + + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Disposition", `attachment; filename="../../named.txt"`) + w.Header().Set("Content-Length", "5") + _, _ = w.Write([]byte("hello")) + })) + t.Cleanup(origin.Close) + + u, err := url.Parse(origin.URL + "/file") + require.NoError(t, err) + require.NoError(t, New(s, DownloadRequest{Directory: tt.directory, URL: u, FileName: "file", UseHeader: true}).Execute()) + + root := s.Filesystem().Path() + require.FileExists(t, filepath.Join(root, tt.want)) + require.NoFileExists(t, filepath.Join(filepath.Dir(root), "named.txt")) + }) + } +} + +// A download that fails to connect is reported as a failed download. +func TestDownloadConnectionFailureIsDownloadError(t *testing.T) { + s := newPullTestServer(t) + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := l.Addr().String() + require.NoError(t, l.Close()) + + u, err := url.Parse("http://" + addr + "/file") + require.NoError(t, err) + err = New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute() + require.Error(t, err) + require.True(t, IsDownloadError(err), "got %v", err) + require.ErrorIs(t, err, ErrDownloadFailed) +} diff --git a/router/downloader/downloader_test.go b/router/downloader/downloader_test.go new file mode 100644 index 000000000..80ccb962e --- /dev/null +++ b/router/downloader/downloader_test.go @@ -0,0 +1,209 @@ +package downloader + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/server" +) + +func TestMain(m *testing.M) { + config.Set(&config.Configuration{AuthenticationToken: "test-token"}) + os.Exit(m.Run()) +} + +// setAllowlist sets the remote download allowlist for the duration of the test. +func setAllowlist(t *testing.T, entries ...string) { + t.Helper() + previous := config.Get() + t.Cleanup(func() { config.Set(previous) }) + next := *previous + next.Api.RemoteDownloadAllowlist = entries + config.Set(&next) +} + +// countingListener counts the connections it accepts. +type countingListener struct { + net.Listener + accepted atomic.Int64 +} + +func (l *countingListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err == nil { + l.accepted.Add(1) + } + return c, err +} + +func TestDownloadDoesNotConnectToInternalAddresses(t *testing.T) { + inner, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + t.Fatal(err) + } + l := &countingListener{Listener: inner} + var requests atomic.Int64 + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("internal")) + })} + go func() { _ = srv.Serve(l) }() + t.Cleanup(func() { _ = srv.Close() }) + port := strconv.Itoa(inner.Addr().(*net.TCPAddr).Port) + + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + + // The shared address space and benchmarking ranges are not reachable on most + // machines, but the download must be refused before any connection is made + // whether or not they are. + for _, host := range []string{"127.0.0.1", "[::1]", "[::ffff:127.0.0.1]", "100.64.0.5", "198.18.0.5", "0.0.0.0"} { + t.Run(host, func(t *testing.T) { + u, _ := url.Parse("http://" + host + ":" + port + "/file") + err := New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute() + if !errors.Is(err, ErrInternalResolution) { + t.Fatalf("expected download from %s to be refused, got %v", host, err) + } + }) + } + + // Give any connection that was made, and any request sent over it, time to + // arrive before checking. + time.Sleep(time.Millisecond * 100) + if n := l.accepted.Load(); n != 0 { + t.Fatalf("expected no connections to internal addresses, got %d", n) + } + if n := requests.Load(); n != 0 { + t.Fatalf("expected no requests to internal addresses, got %d", n) + } +} + +// The address is checked again immediately before connecting, since the host is +// resolved again when dialing and could resolve to an internal address then. +func TestCheckConnectionRefusesInternalAddresses(t *testing.T) { + for _, addr := range []string{"127.0.0.1:80", "[::1]:80", "10.0.0.1:443", "100.64.0.1:80", "[fe80::1%lo]:80", "[64:ff9b::7f00:1]:80"} { + if err := checkConnection(context.Background(), "tcp", addr, nil); !errors.Is(err, ErrInternalResolution) { + t.Errorf("expected connection to %s to be refused, got %v", addr, err) + } + } + for _, addr := range []string{"1.1.1.1:443", "[2606:4700:4700::1111]:443"} { + if err := checkConnection(context.Background(), "tcp", addr, nil); err != nil { + t.Errorf("expected connection to %s to be allowed, got %v", addr, err) + } + } +} + +// Operators can allow destinations that are otherwise refused, such as services +// on a Tailscale network or a cloud provider's internal endpoints. +func TestRemoteDownloadAllowlist(t *testing.T) { + setAllowlist(t, "100.64.0.0/10", "internal.example.", "127.0.0.1") + + host := func(h string) context.Context { return context.WithValue(context.Background(), dialHostKey{}, h) } + tests := []struct { + ctx context.Context + addr string + allowed bool + }{ + {context.Background(), "100.64.0.1:80", true}, + {context.Background(), "100.127.255.254:443", true}, + {context.Background(), "10.0.0.1:80", false}, + {host("internal.example"), "10.0.0.1:80", true}, + {host("other.example"), "10.0.0.1:80", false}, + {context.Background(), "1.1.1.1:443", true}, + } + for _, tt := range tests { + err := checkConnection(tt.ctx, "tcp", tt.addr, nil) + if tt.allowed && err != nil { + t.Errorf("expected connection to %s to be allowed, got %v", tt.addr, err) + } else if !tt.allowed && !errors.Is(err, ErrInternalResolution) { + t.Errorf("expected connection to %s to be refused, got %v", tt.addr, err) + } + } + + // A request to an allowed internal address is made through the client used + // for downloads. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("allowed")) + })} + go func() { _ = srv.Serve(l) }() + t.Cleanup(func() { _ = srv.Close() }) + res, err := client.Get("http://" + l.Addr().String() + "/file") + if err != nil { + t.Fatalf("expected a request to an allowed address to succeed, got %v", err) + } + b, _ := io.ReadAll(res.Body) + _ = res.Body.Close() + if string(b) != "allowed" { + t.Fatalf("expected the response from the allowed address, got %q", b) + } +} + +// Hosts that cannot be resolved are refused. +func TestDownloadRefusesUnresolvableHosts(t *testing.T) { + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + u, _ := url.Parse("http://does-not-exist.invalid/file") + if err := New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute(); !IsDownloadError(err) { + t.Fatalf("expected an unresolvable host to be refused, got %v", err) + } +} + +// Downloads from internal addresses are refused, whether or not anything is +// listening on the port. +func TestDownloadsFromInternalPortsAreRefused(t *testing.T) { + var wg sync.WaitGroup + open, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer open.Close() + wg.Add(1) + go func() { + defer wg.Done() + if c, err := open.Accept(); err == nil { + _ = c.Close() + } + }() + closed, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + closedAddr := closed.Addr().String() + _ = closed.Close() + + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + var errs []error + for _, addr := range []string{open.Addr().String(), closedAddr} { + u, _ := url.Parse("http://" + addr + "/file") + errs = append(errs, New(s, DownloadRequest{Directory: "/", URL: u, FileName: "file"}).Execute()) + } + for _, err := range errs { + if !errors.Is(err, ErrInternalResolution) { + t.Fatalf("expected downloads from open and closed ports to be refused, got %v", errs) + } + } + _ = open.Close() + wg.Wait() +} diff --git a/router/downloader/max_size_reader_test.go b/router/downloader/max_size_reader_test.go new file mode 100644 index 000000000..617343657 --- /dev/null +++ b/router/downloader/max_size_reader_test.go @@ -0,0 +1,24 @@ +package downloader + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestMaxSizeReaderAllowsExactLimit(t *testing.T) { + m := &maxSizeReader{r: bytes.NewReader(make([]byte, 10)), remaining: 10} + n, err := io.Copy(io.Discard, m) + if err != nil || n != 10 || m.exceeded { + t.Fatalf("expected 10 bytes without error, got n=%d err=%v exceeded=%v", n, err, m.exceeded) + } +} + +func TestMaxSizeReaderRejectsOversizedBody(t *testing.T) { + m := &maxSizeReader{r: bytes.NewReader(make([]byte, 11)), remaining: 10} + _, err := io.Copy(io.Discard, m) + if !errors.Is(err, ErrFileTooLarge) || !m.exceeded { + t.Fatalf("expected ErrFileTooLarge, got err=%v exceeded=%v", err, m.exceeded) + } +} diff --git a/router/middleware/middleware.go b/router/middleware/middleware.go index 43cdcc01d..7b888277d 100644 --- a/router/middleware/middleware.go +++ b/router/middleware/middleware.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "strings" + "time" "emperror.dev/errors" "github.com/apex/log" @@ -57,7 +58,7 @@ func AttachApiClient(client remote.Client) gin.HandlerFunc { // at the time it is called the stack will be attached. func CaptureAndAbort(c *gin.Context, err error) { c.Abort() - c.Error(errors.WithStackDepthIf(err, 1)) + _ = c.Error(errors.WithStackDepthIf(err, 1)) } // CaptureErrors is custom handler function allowing for errors bubbled up by @@ -105,7 +106,7 @@ func SetAccessControlHeaders() gin.HandlerFunc { // CORS for Private Networks (RFC1918) // @see https://developer.chrome.com/blog/private-network-access-update/?utm_source=devtools if allowPrivateNetwork { - c.Header("Access-Control-Request-Private-Network", "true") + c.Header("Access-Control-Allow-Private-Network", "true") } // Maximum age allowable under Chromium v76 is 2 hours, so just use that since @@ -182,10 +183,20 @@ func RequireAuthorization() gin.HandlerFunc { c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "You are not authorized to access this endpoint."}) return } + ClearDeadlines(c) c.Next() } } +// ClearDeadlines removes the deadlines the server applies to requests that have +// not authenticated, so that an authenticated request is able to send and +// receive large bodies, such as file uploads, downloads and server transfers. +func ClearDeadlines(c *gin.Context) { + rc := http.NewResponseController(c.Writer) + _ = rc.SetReadDeadline(time.Time{}) + _ = rc.SetWriteDeadline(time.Time{}) +} + // RemoteDownloadEnabled checks if remote downloads are enabled for this instance // and if not aborts the request. func RemoteDownloadEnabled() gin.HandlerFunc { diff --git a/router/middleware/middleware_cors_test.go b/router/middleware/middleware_cors_test.go new file mode 100644 index 000000000..09e1cd0ea --- /dev/null +++ b/router/middleware/middleware_cors_test.go @@ -0,0 +1,45 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" +) + +func newCORSTestEngine(t *testing.T, configure func(cfg *config.Configuration)) *gin.Engine { + t.Helper() + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.PanelLocation = "http://panel.test" + configure(cfg) + config.Set(cfg) + + gin.SetMode(gin.ReleaseMode) + engine := gin.New() + engine.Use(SetAccessControlHeaders()) + engine.GET("/resource", func(c *gin.Context) { c.Status(http.StatusOK) }) + return engine +} + +// The private network grant is sent under the response header name browsers +// read. +func TestPrivateNetworkPreflightGrantUsesResponseHeader(t *testing.T) { + engine := newCORSTestEngine(t, func(cfg *config.Configuration) { + cfg.AllowCORSPrivateNetwork = true + }) + + req := httptest.NewRequest(http.MethodOptions, "/resource", nil) + req.Header.Set("Origin", "http://panel.test") + req.Header.Set("Access-Control-Request-Private-Network", "true") + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + + require.Equal(t, "true", rec.Header().Get("Access-Control-Allow-Private-Network")) +} diff --git a/router/middleware/request_error.go b/router/middleware/request_error.go index 96ce93158..6dedfe656 100644 --- a/router/middleware/request_error.go +++ b/router/middleware/request_error.go @@ -10,6 +10,7 @@ import ( "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/Rene-Roscher/wings/internal/ufs" "github.com/Rene-Roscher/wings/server" "github.com/Rene-Roscher/wings/server/filesystem" ) @@ -128,6 +129,9 @@ func (re *RequestError) asFilesystemError() (int, string) { if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) || strings.Contains(err.Error(), "filesystem: is a directory") { return http.StatusBadRequest, "Cannot perform that action: file is a directory." } + if errors.Is(err, ufs.ErrNotRegular) { + return http.StatusBadRequest, "Cannot perform that action: not a regular file." + } if filesystem.IsErrorCode(err, filesystem.ErrCodeDiskSpace) || strings.Contains(err.Error(), "filesystem: not enough disk space") { return http.StatusBadRequest, "There is not enough disk space available to perform that action." } diff --git a/router/router.go b/router/router.go index 15c72da29..0aed3bc61 100644 --- a/router/router.go +++ b/router/router.go @@ -1,7 +1,10 @@ package router import ( + "crypto/tls" + "net/http" "regexp" + "time" "emperror.dev/errors" "github.com/apex/log" @@ -15,6 +18,44 @@ import ( var tokenRegex = regexp.MustCompile(`([?|&]token=)([^&]+)($|&)`) +// Timeouts for the HTTP server. +var ( + // ReadHeaderTimeout is the time a client has to send the request headers. + ReadHeaderTimeout = 10 * time.Second + // IdleTimeout is the time an idle keep-alive connection is kept open. + IdleTimeout = 2 * time.Minute + // unauthenticatedTimeout is the time a client has to send the rest of its + // request, and to read the response, if it has not authenticated. + unauthenticatedTimeout = 30 * time.Second +) + +// NewServer returns an HTTP server for the handler with the timeouts used by +// Wings. +func NewServer(addr string, handler http.Handler, tlsConfig *tls.Config) *http.Server { + return &http.Server{ + Addr: addr, + Handler: limitUnauthenticated(handler, unauthenticatedTimeout), + TLSConfig: tlsConfig, + ReadHeaderTimeout: ReadHeaderTimeout, + IdleTimeout: IdleTimeout, + // Pass "OPTIONS *" requests to the handler like any other request. + DisableGeneralOptionsHandler: true, + } +} + +// limitUnauthenticated sets a deadline for reading the request and writing the +// response before any other code handles the request. middleware.ClearDeadlines +// removes the deadline once a request is authenticated. +func limitUnauthenticated(next http.Handler, timeout time.Duration) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rc := http.NewResponseController(w) + deadline := time.Now().Add(timeout) + _ = rc.SetReadDeadline(deadline) + _ = rc.SetWriteDeadline(deadline) + next.ServeHTTP(w, r) + }) +} + // Configure configures the routing infrastructure for this daemon instance. func Configure(m *wserver.Manager, client remote.Client) *gin.Engine { gin.SetMode("release") @@ -23,7 +64,6 @@ func Configure(m *wserver.Manager, client remote.Client) *gin.Engine { router.Use(gin.Recovery()) if err := router.SetTrustedProxies(config.Get().Api.TrustedProxies); err != nil { panic(errors.WithStack(err)) - return nil } router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders()) router.Use(middleware.AttachServerManager(m), middleware.AttachApiClient(client)) @@ -51,7 +91,7 @@ func Configure(m *wserver.Manager, client remote.Client) *gin.Engine { // This route is special it sits above all the other requests because we are // using a JWT to authorize access to it, therefore it needs to be publicly // accessible. - router.GET("/api/servers/:server/ws", middleware.ServerExists(), getServerWebsocket) + router.GET("/api/servers/:server/ws", middleware.ServerExists(), getServerWebsocket(maxWebsocketConnections, websocketAuthenticationTimeout)) // This request is called by another daemon when a server is going to be transferred out. // This request does not need the AuthorizationMiddleware as the panel should never call it diff --git a/router/router_deauthorize_test.go b/router/router_deauthorize_test.go new file mode 100644 index 000000000..803bb1fb2 --- /dev/null +++ b/router/router_deauthorize_test.go @@ -0,0 +1,107 @@ +package router + +import ( + "bytes" + "encoding/json" + "errors" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/gbrlsnchs/jwt/v3" + ws "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/router/tokens" + "github.com/Rene-Roscher/wings/router/websocket" +) + +// authenticationMessageFor returns an authentication message carrying a token +// issued to the given user for the test server. +func authenticationMessageFor(t *testing.T, userUUID string) websocket.Message { + t.Helper() + now := time.Now().Add(time.Second * 2) + payload := tokens.WebsocketPayload{ + Payload: jwt.Payload{ + IssuedAt: jwt.NumericDate(now), + ExpirationTime: jwt.NumericDate(now.Add(time.Minute * 10)), + }, + Scoped: tokens.Scoped{Scope: string(tokens.Websocket)}, + UserUUID: userUUID, + ServerUUID: websocketTestServerUuid, + Permissions: []string{websocket.PermissionConnect}, + } + token, err := jwt.Sign(&payload, config.GetJwtAlgorithm()) + require.NoError(t, err) + return websocket.Message{Event: websocket.AuthenticationEvent, Args: []string{string(token)}} +} + +// authenticateAs sends a token for the given user and reports whether the +// server accepted it. +func authenticateAs(t *testing.T, conn *ws.Conn, userUUID string) bool { + t.Helper() + if err := conn.WriteJSON(authenticationMessageFor(t, userUUID)); err != nil { + return false + } + _ = conn.SetReadDeadline(time.Now().Add(time.Second * 5)) + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + for { + var m websocket.Message + if err := conn.ReadJSON(&m); err != nil { + return false + } + switch m.Event { + case websocket.AuthenticationSuccessEvent: + return true + case websocket.ErrorEvent, websocket.JwtErrorEvent: + return false + } + } +} + +// Revoking one user's access to a server closes that user's sockets and leaves +// other users' sockets on the server connected. +func TestDeauthorizeUserLeavesOtherUsersWebsocketsOpen(t *testing.T) { + url := startWebsocketTestServer(t) + const revokedUser = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" + const otherUser = "ffffffff-ffff-4fff-8fff-ffffffffffff" + + revokedConn, err := dialWebsocket(t, url) + require.NoError(t, err) + require.True(t, authenticateAs(t, revokedConn, revokedUser)) + + otherConn, err := dialWebsocket(t, url) + require.NoError(t, err) + require.True(t, authenticateAs(t, otherConn, otherUser)) + + base := url[:strings.Index(url, "/api/")] + body, err := json.Marshal(map[string]any{"user": revokedUser, "servers": []string{websocketTestServerUuid}}) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, "http"+strings.TrimPrefix(base, "ws")+"/api/deauthorize-user", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer test-token") + req.Header.Set("Content-Type", "application/json") + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + _ = res.Body.Close() + require.Equal(t, http.StatusNoContent, res.StatusCode) + + // The revoked user's socket is closed. + require.NoError(t, revokedConn.SetReadDeadline(time.Now().Add(3*time.Second))) + for { + if _, _, err := revokedConn.ReadMessage(); err != nil { + var ne net.Error + require.False(t, errors.As(err, &ne) && ne.Timeout(), "expected the revoked user's socket to be closed") + break + } + } + + // Sockets are closed in the background, so allow time for that. + time.Sleep(200 * time.Millisecond) + + // The other user's socket is still open and can be used. + require.True(t, authenticateAs(t, otherConn, otherUser), "revoking one user should not close another user's socket on the same server") +} diff --git a/router/router_download.go b/router/router_download.go index 2a6d2ab24..d370cf3d5 100644 --- a/router/router_download.go +++ b/router/router_download.go @@ -34,6 +34,7 @@ func getDownloadBackup(c *gin.Context) { }) return } + middleware.ClearDeadlines(c) // Validate that the BackupUuid field is actually a UUID and not some random characters or a // file path. @@ -88,6 +89,7 @@ func getDownloadFile(c *gin.Context) { }) return } + middleware.ClearDeadlines(c) f, st, err := s.Filesystem().File(token.FilePath) if err != nil { diff --git a/router/router_server.go b/router/router_server.go index 108a14b71..caa7f9957 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -207,6 +207,7 @@ func deleteServer(c *gin.Context) { s.Events().Publish(server.DeletedEvent, nil) s.CleanupForDestroy() + s.CancelBackupOperations() // Remove any pending remote file downloads for the server. for _, dl := range downloader.ByServer(s.ID()) { diff --git a/router/router_server_backup.go b/router/router_server_backup.go index dcdd989b3..36c6057d1 100644 --- a/router/router_server_backup.go +++ b/router/router_server_backup.go @@ -20,22 +20,16 @@ import ( "github.com/Rene-Roscher/wings/config" "github.com/Rene-Roscher/wings/environment" + wnet "github.com/Rene-Roscher/wings/internal/network" "github.com/Rene-Roscher/wings/router/middleware" "github.com/Rene-Roscher/wings/server" "github.com/Rene-Roscher/wings/server/backup" + "github.com/Rene-Roscher/wings/server/filesystem" ) // isValidBackupContentType is now replaced by backup.IsValidBackupContentType // which uses the extensible CompressionRegistry for better format support -// blockedBackupRestorePrefixes lists IP ranges that backup restore downloads are -// never allowed to reach (in addition to private/loopback/link-local ranges), -// unless a destination is explicitly permitted via the RestoreHostAllowlist. -var blockedBackupRestorePrefixes = []netip.Prefix{ - netip.MustParsePrefix("100.64.0.0/10"), - netip.MustParsePrefix("198.18.0.0/15"), -} - // backupDownloadError is a sentinel error type used to surface backup download // validation failures (e.g. SSRF protection) back to the API caller as 400s. type backupDownloadError string @@ -83,6 +77,11 @@ func postServerBackup(c *gin.Context) { return } + if err := filesystem.ValidateIgnore(data.Ignore); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + var adapter backup.BackupInterface switch data.Adapter { case backup.LocalBackupAdapter: @@ -115,15 +114,23 @@ func postServerBackup(c *gin.Context) { // ATOMIC REGISTRATION: Register and get accurate queue status registry := server.GetBackupOperationRegistry() logger.Info("registering backup operation in queue system") - + // ATOMIC: Register operation and get queue status atomically _, ctx, cancel, err, wasQueued := registry.Register(s.Context(), backupUuid, s.ID(), server.OperationTypeBackup) if err != nil { logger.WithError(err).Error("failed to register backup operation") - s.Events().Publish(server.DaemonMessageEvent, "Failed to register backup: " + err.Error()) + s.Events().Publish(server.DaemonMessageEvent, "Failed to register backup: "+err.Error()) return } - + // Defer cleanup immediately after a successful registration so that the + // registry slot is always released, even if anything below panics. + defer func() { + logger.Debug("backup goroutine cleanup starting") + registry.Complete(backupUuid) + cancel() // Cancel AFTER marking complete + logger.Debug("backup goroutine cleanup completed") + }() + // ACCURATE STATE MANAGEMENT: Set state based on actual queue experience if wasQueued { s.Environment.SetState(environment.ProcessBackupQueuedState) @@ -131,13 +138,6 @@ func postServerBackup(c *gin.Context) { } else { s.Events().Publish(server.DaemonMessageEvent, "Backup slot available - starting backup process immediately...") } - // Defer cleanup - will run AFTER backup completes - defer func() { - logger.Debug("backup goroutine cleanup starting") - registry.Complete(backupUuid) - cancel() // Cancel AFTER marking complete - logger.Debug("backup goroutine cleanup completed") - }() // Add timeout if not already set ctx, timeoutCancel := context.WithTimeout(ctx, 6*time.Hour) @@ -145,7 +145,7 @@ func postServerBackup(c *gin.Context) { if err := s.BackupWithRetry(ctx, b, 2); err != nil { logger.WithField("error", errors.WithStackIf(err)).Error("router: failed to generate server backup after retries") - + // Send failure event to ensure frontend gets notified s.Events().Publish(server.BackupCompletedEvent, map[string]any{ "uuid": backupUuid, @@ -167,8 +167,6 @@ func postServerBackup(c *gin.Context) { // // This endpoint will block until the backup is fully restored allowing for a // spinner to be displayed in the Panel UI effectively. -// -// TODO: stop the server if it is running func postServerRestoreBackup(c *gin.Context) { s := middleware.ExtractServer(c) client := middleware.ExtractApiClient(c) @@ -223,16 +221,10 @@ func postServerRestoreBackup(c *gin.Context) { // to prevent race conditions with the operation registry logger.Info("processing server backup restore request") - if data.TruncateDirectory { - logger.Info("received \"truncate_directory\" flag in request: deleting server files") - if err := s.Filesystem().TruncateRootDirectory(); err != nil { - middleware.CaptureAndAbort(c, err) - return - } - } - // Now that we've cleaned up the data directory if necessary, grab the backup file - // and attempt to restore it into the server directory. + // Find the backup before changing anything, so that a restore that cannot + // happen leaves the server's files as they are. If the truncate_directory flag + // is set, the files are deleted once the server has stopped. if data.Adapter == backup.LocalBackupAdapter { b, _, err := backup.LocateLocal(client, backupUuid) if err != nil { @@ -240,25 +232,25 @@ func postServerRestoreBackup(c *gin.Context) { return } go func(s *server.Server, b backup.BackupInterface, logger *log.Entry) { + // Never let a panic in this goroutine crash the daemon. + defer func() { + if r := recover(); r != nil { + logger.WithField("panic", r).Error("local restore goroutine panicked") + } + }() // ATOMIC REGISTRATION: Register and get accurate queue status registry := server.GetBackupOperationRegistry() logger.Info("registering local restore operation in queue system") - + // ATOMIC: Register operation and get queue status atomically _, ctx, cancel, err, wasQueued := registry.Register(s.Context(), backupUuid, s.ID(), server.OperationTypeRestore) if err != nil { logger.WithError(err).Error("failed to register restore operation") - s.Events().Publish(server.DaemonMessageEvent, "Failed to register restore: " + err.Error()) + s.Events().Publish(server.DaemonMessageEvent, "Failed to register restore: "+err.Error()) return } - - // ACCURATE STATE MANAGEMENT: Set state based on actual queue experience - if wasQueued { - s.Environment.SetState(environment.ProcessRestoreQueuedState) - s.Events().Publish(server.DaemonMessageEvent, "Restore was queued and slot acquired - starting restore process...") - } else { - s.Events().Publish(server.DaemonMessageEvent, "Restore slot available - starting restore process immediately...") - } + // Defer cleanup immediately after a successful registration so that the + // registry slot is always released, even if anything below panics. defer func() { if r := recover(); r != nil { logger.WithField("panic", r).Error("restore operation panicked") @@ -270,21 +262,29 @@ func postServerRestoreBackup(c *gin.Context) { // Note: SetRestoring is now handled atomically within the restore function }() + // ACCURATE STATE MANAGEMENT: Set state based on actual queue experience + if wasQueued { + s.Environment.SetState(environment.ProcessRestoreQueuedState) + s.Events().Publish(server.DaemonMessageEvent, "Restore was queued and slot acquired - starting restore process...") + } else { + s.Events().Publish(server.DaemonMessageEvent, "Restore slot available - starting restore process immediately...") + } + // Add 4-hour timeout for restore operations ctx, timeoutCancel := context.WithTimeout(ctx, 4*time.Hour) defer timeoutCancel() logger.Info("starting restoration process for server backup using local driver") - if err := s.RestoreBackupWithContext(ctx, b, nil); err != nil { + if err := s.RestoreBackupWithContext(ctx, b, nil, data.TruncateDirectory); err != nil { logger.WithField("error", err).Error("failed to restore local backup to server") - s.Events().Publish(server.DaemonMessageEvent, "Failed server restoration from local backup: " + err.Error()) + s.Events().Publish(server.DaemonMessageEvent, "Failed server restoration from local backup: "+err.Error()) // BackupRestoreCompletedEvent is now sent by RestoreBackupWithContext } else { logger.WithFields(log.Fields{ "is_restoring": s.IsRestoring(), "server_state": s.Environment.State(), }).Info("Local restore completed successfully") - + s.Events().Publish(server.DaemonMessageEvent, "Completed server restoration from local backup.") // BackupRestoreCompletedEvent is now sent by RestoreBackupWithContext logger.Info("completed server restoration from local backup") @@ -299,26 +299,29 @@ func postServerRestoreBackup(c *gin.Context) { // parse over the contents as we go in order to restore it to the server. // // backupRestoreHttpClient enforces SSRF protections: it refuses to connect to - // private/internal/loopback addresses (and the explicitly blocked ranges) unless + // private/internal/loopback addresses (see internal/network) unless // the destination is permitted via the RestoreHostAllowlist configuration option. httpClient := backupRestoreHttpClient() - logger.WithField("download_url", data.DownloadUrl).Info("downloading backup from remote location...") - // Use proper timeout to prevent indefinite hangs during backup downloads. - // 2 hour timeout should be sufficient for most backup file sizes while preventing - // resource exhaustion from stuck connections. + // Only log the host: the download URL is presigned and its query string + // carries the signature. + downloadHost := "" + if u, err := url.Parse(data.DownloadUrl); err == nil { + downloadHost = u.Host + } + logger.WithField("download_host", downloadHost).Info("downloading backup from remote location...") req, err := http.NewRequestWithContext(s.Context(), http.MethodGet, data.DownloadUrl, nil) if err != nil { logger.WithField("error", err).Error("failed to create HTTP request for backup download") middleware.CaptureAndAbort(c, err) return } - + logger.Debug("executing HTTP request for backup download") downloadStart := time.Now() res, err := httpClient.Do(req) if err != nil { logger.WithFields(log.Fields{ - "error": err, + "error": err, "duration_ms": time.Since(downloadStart).Milliseconds(), }).Error("HTTP request failed for backup download") // Surface SSRF/validation failures from the restore HTTP client as a 400 @@ -331,14 +334,14 @@ func postServerRestoreBackup(c *gin.Context) { middleware.CaptureAndAbort(c, err) return } - + logger.WithFields(log.Fields{ - "status_code": res.StatusCode, + "status_code": res.StatusCode, "content_length": res.ContentLength, - "content_type": res.Header.Get("Content-Type"), - "duration_ms": time.Since(downloadStart).Milliseconds(), + "content_type": res.Header.Get("Content-Type"), + "duration_ms": time.Since(downloadStart).Milliseconds(), }).Info("received HTTP response for backup download") - + // CRITICAL: Ensure response body is always closed in error paths before goroutine takes ownership var goroutineStarted bool defer func() { @@ -368,33 +371,33 @@ func postServerRestoreBackup(c *gin.Context) { }) return } - + // Mark that goroutine will take ownership of the response goroutineStarted = true go func(s *server.Server, uuid string, logger *log.Entry) { // CRITICAL: Always close response body to prevent resource leak defer res.Body.Close() - + // Never let a panic in this goroutine crash the daemon. + defer func() { + if r := recover(); r != nil { + logger.WithField("panic", r).Error("S3 restore goroutine panicked") + } + }() + // ATOMIC REGISTRATION: Register and get accurate queue status registry := server.GetBackupOperationRegistry() logger.Info("registering S3 restore operation in queue system") - + // ATOMIC: Register operation and get queue status atomically _, ctx, cancel, err, wasQueued := registry.Register(s.Context(), uuid, s.ID(), server.OperationTypeRestore) if err != nil { logger.WithError(err).Error("failed to register S3 restore operation") - s.Events().Publish(server.DaemonMessageEvent, "Failed to register S3 restore: " + err.Error()) + s.Events().Publish(server.DaemonMessageEvent, "Failed to register S3 restore: "+err.Error()) return } - - // ACCURATE STATE MANAGEMENT: Set state based on actual queue experience - if wasQueued { - s.Environment.SetState(environment.ProcessRestoreQueuedState) - s.Events().Publish(server.DaemonMessageEvent, "S3 restore was queued and slot acquired - starting restore process...") - } else { - s.Events().Publish(server.DaemonMessageEvent, "S3 restore slot available - starting restore process immediately...") - } + // Defer cleanup immediately after a successful registration so that the + // registry slot is always released, even if anything below panics. defer func() { if r := recover(); r != nil { logger.WithField("panic", r).Error("S3 restore operation panicked") @@ -406,20 +409,28 @@ func postServerRestoreBackup(c *gin.Context) { // Note: SetRestoring is now handled atomically within the restore function }() + // ACCURATE STATE MANAGEMENT: Set state based on actual queue experience + if wasQueued { + s.Environment.SetState(environment.ProcessRestoreQueuedState) + s.Events().Publish(server.DaemonMessageEvent, "S3 restore was queued and slot acquired - starting restore process...") + } else { + s.Events().Publish(server.DaemonMessageEvent, "S3 restore slot available - starting restore process immediately...") + } + // Add 4-hour timeout for restore operations ctx, timeoutCancel := context.WithTimeout(ctx, 4*time.Hour) defer timeoutCancel() logger.WithField("content_length", res.ContentLength).Info("starting restoration process for server backup using S3 driver") - + // Create S3 backup instance s3Backup := backup.NewS3(client, uuid, "") - + // Wrap response body with download progress tracking if we know the size - var downloadReader io.ReadCloser = res.Body + downloadReader := io.ReadCloser(res.Body) if res.ContentLength > 0 { logger.WithField("size_mb", res.ContentLength/(1024*1024)).Info("S3 backup download size known, adding download progress tracking") - + // Progress callback for download tracking - send WebSocket events! onProgress := func(downloaded, total int64) { // Calculate percentage for download phase @@ -427,7 +438,7 @@ func postServerRestoreBackup(c *gin.Context) { if total > 0 { percentage = int((downloaded * 100) / total) } - + // Send WebSocket event for download progress // This gives immediate feedback to the user s.Events().Publish(server.BackupProgressEvent, server.BackupProgressUpdate{ @@ -437,35 +448,35 @@ func postServerRestoreBackup(c *gin.Context) { BytesWritten: downloaded, BytesTotal: total, }) - + // Also log for debugging if percentage%10 == 0 || downloaded == total { logger.WithFields(log.Fields{ "downloaded_percentage": percentage, - "downloaded_mb": downloaded / (1024 * 1024), - "total_mb": total / (1024 * 1024), + "downloaded_mb": downloaded / (1024 * 1024), + "total_mb": total / (1024 * 1024), }).Info("S3 download progress") } } - + downloadReader = backup.NewDownloadProgressReader(res.Body, res.ContentLength, uuid, onProgress) } - + // Pass download size through context for accurate restore progress if res.ContentLength > 0 { - ctx = context.WithValue(ctx, "download_size", res.ContentLength) + ctx = context.WithValue(ctx, server.DownloadSizeContextKey, res.ContentLength) } - - if err := s.RestoreBackupWithContext(ctx, s3Backup, downloadReader); err != nil { + + if err := s.RestoreBackupWithContext(ctx, s3Backup, downloadReader, data.TruncateDirectory); err != nil { logger.WithField("error", errors.WithStack(err)).Error("failed to restore remote S3 backup to server") - s.Events().Publish(server.DaemonMessageEvent, "Failed server restoration from S3 backup: " + err.Error()) + s.Events().Publish(server.DaemonMessageEvent, "Failed server restoration from S3 backup: "+err.Error()) // BackupRestoreCompletedEvent is now sent by RestoreBackupWithContext } else { logger.WithFields(log.Fields{ "is_restoring": s.IsRestoring(), "server_state": s.Environment.State(), }).Info("S3 restore completed successfully") - + s.Events().Publish(server.DaemonMessageEvent, "Completed server restoration from S3 backup.") // BackupRestoreCompletedEvent is now sent by RestoreBackupWithContext logger.Info("completed server restoration from S3 backup") @@ -487,7 +498,7 @@ func deleteServerBackup(c *gin.Context) { // UNIFIED BEHAVIOR: Try to locate and delete backup regardless of type (Local or S3) // This ensures consistent deletion behavior between storage types (WORK.md requirement) - + // First try to locate as Local backup if localBackup, _, err := backup.LocateLocal(client, backupID); err == nil { // Found as Local backup - delete it @@ -498,7 +509,7 @@ func deleteServerBackup(c *gin.Context) { c.Status(http.StatusNoContent) return } - + // If not found as Local backup, check if it's an S3 backup file that exists locally // S3 backups may leave local files behind after failed uploads or for debugging s3Backup := backup.NewS3(client, backupID, "") @@ -511,7 +522,7 @@ func deleteServerBackup(c *gin.Context) { c.Status(http.StatusNoContent) return } - + // If neither Local nor S3 backup file found, return 404 c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The requested backup was not found on this server.", @@ -690,41 +701,16 @@ func isBlockedBackupRestoreIP(host string, ip net.IP) bool { return true } addr = addr.Unmap() - if !addr.IsGlobalUnicast() || addr.IsPrivate() || addr.IsLoopback() || addr.IsLinkLocalUnicast() || isExplicitlyBlockedBackupRestoreIP(addr) { + if wnet.IsInternal(addr) { return !isAllowedBackupRestoreDestination(host, addr) } return false } -func isExplicitlyBlockedBackupRestoreIP(addr netip.Addr) bool { - for _, prefix := range blockedBackupRestorePrefixes { - if prefix.Contains(addr) { - return true - } - } - return false -} - // isAllowedBackupRestoreDestination reports whether the given host/addr is explicitly // permitted through the System.Backups.RestoreHostAllowlist configuration option. func isAllowedBackupRestoreDestination(host string, addr netip.Addr) bool { - host = strings.TrimSuffix(strings.ToLower(host), ".") - for _, entry := range config.Get().System.Backups.RestoreHostAllowlist { - entry = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(entry)), ".") - if entry == "" { - continue - } - if entry == host { - return true - } - if allowedAddr, err := netip.ParseAddr(entry); err == nil && allowedAddr.Unmap() == addr { - return true - } - if prefix, err := netip.ParsePrefix(entry); err == nil && prefix.Contains(addr) { - return true - } - } - return false + return wnet.IsAllowed(config.Get().System.Backups.RestoreHostAllowlist, host, addr) } // isSupportedBackupRestoreContentType reports whether the given Content-Type header diff --git a/router/router_server_backup_test.go b/router/router_server_backup_test.go index 1662c04b3..c909a63ac 100644 --- a/router/router_server_backup_test.go +++ b/router/router_server_backup_test.go @@ -19,6 +19,7 @@ import ( "github.com/Rene-Roscher/wings/internal/models" "github.com/Rene-Roscher/wings/remote" wserver "github.com/Rene-Roscher/wings/server" + "github.com/Rene-Roscher/wings/server/filesystem" ) func init() { @@ -197,6 +198,23 @@ func TestPostServerRestoreBackupRejectsLoopbackDownloadURL(t *testing.T) { } } +func TestPostServerBackupRejectsInvalidIgnoreList(t *testing.T) { + client := backupTestRemoteClient{} + backupID := "11111111-1111-1111-1111-111111111111" + ignore := strings.Repeat("*a", filesystem.MaxIgnorePatternWildcards+1) + c, w, s := newBackupRestoreContext(t, client, backupID, fmt.Sprintf(`{"adapter":"wings","uuid":%q,"ignore":%q}`, backupID, ignore)) + defer s.CtxCancel() + + postServerBackup(c) + + if c.Writer.Status() != http.StatusBadRequest { + t.Fatalf("expected invalid ignore list to be rejected, got status %d body %s", c.Writer.Status(), w.Body.String()) + } + if !strings.Contains(w.Body.String(), "wildcards") { + t.Fatalf("expected error to describe the wildcard limit, got body %s", w.Body.String()) + } +} + func TestPostServerRestoreBackupRejectsNonUuidBackupID(t *testing.T) { client := backupTestRemoteClient{restoreStatus: make(chan string, 1)} c, w, s := newBackupRestoreContext(t, client, "../target/archive", `{"adapter":"s3","download_url":"https://example.com/archive.tar.gz"}`) diff --git a/router/router_server_files.go b/router/router_server_files.go index 7a2936b90..7df668963 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -4,7 +4,6 @@ import ( "bufio" "context" "io" - "mime/multipart" "net/http" "net/url" "os" @@ -640,8 +639,11 @@ func postServerUploadFiles(c *gin.Context) { }) return } + middleware.ClearDeadlines(c) - form, err := c.MultipartForm() + // Read the files one part at a time and write each straight to the server's + // disk. + reader, err := c.Request.MultipartReader() if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Failed to get multipart form data from request.", @@ -649,57 +651,67 @@ func postServerUploadFiles(c *gin.Context) { return } - headers, ok := form.File["files"] - if !ok { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "No files were found on the request body.", - }) - return - } - directory := c.Query("directory") maxFileSize := config.Get().Api.UploadLimit maxFileSizeBytes := maxFileSize * 1024 * 1024 - var totalSize int64 - for _, header := range headers { - if header.Size > maxFileSizeBytes { + var found bool + for { + part, err := reader.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "File " + header.Filename + " is larger than the maximum file upload size of " + strconv.FormatInt(maxFileSize, 10) + " MB.", + "error": "Failed to get multipart form data from request.", }) return } - totalSize += header.Size - } + if part.FormName() != "files" || part.FileName() == "" { + continue + } + found = true - for _, header := range headers { - // We run this in a different method so I can use defer without any of - // the consequences caused by calling it in a loop. - if err := handleFileUpload(filepath.Join(directory, header.Filename), s, header); err != nil { + name := part.FileName() + if err := handleFileUpload(filepath.Join(directory, name), s, part, maxFileSizeBytes); err != nil { + if errors.Is(err, errUploadTooLarge) { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "File " + name + " is larger than the maximum file upload size of " + strconv.FormatInt(maxFileSize, 10) + " MB.", + }) + return + } middleware.CaptureAndAbort(c, err) return - } else { - s.SaveActivity(s.NewRequestActivity(token.UserUuid, c.ClientIP()), server.ActivityFileUploaded, models.ActivityMeta{ - "file": header.Filename, - "directory": filepath.Clean(directory), - }) } + s.SaveActivity(s.NewRequestActivity(token.UserUuid, c.ClientIP()), server.ActivityFileUploaded, models.ActivityMeta{ + "file": name, + "directory": filepath.Clean(directory), + }) } -} -func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) error { - file, err := header.Open() - if err != nil { - return err + if !found { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "No files were found on the request body.", + }) } - defer file.Close() +} + +var errUploadTooLarge = errors.New("router: uploaded file is larger than the upload limit") +// handleFileUpload writes an uploaded file to p, refusing it if it is larger +// than limit bytes. +func handleFileUpload(p string, s *server.Server, r io.Reader, limit int64) error { if err := s.Filesystem().IsIgnored(p); err != nil { return err } - if err := s.Filesystem().Write(p, file, header.Size, 0o644); err != nil { + n, err := s.Filesystem().WriteFrom(p, io.LimitReader(r, limit+1)) + if err != nil { return err } + if n > limit { + _ = s.Filesystem().Delete(p) + return errUploadTooLarge + } return nil } diff --git a/router/router_server_files_upload_test.go b/router/router_server_files_upload_test.go new file mode 100644 index 000000000..556eeec33 --- /dev/null +++ b/router/router_server_files_upload_test.go @@ -0,0 +1,198 @@ +package router + +import ( + "bytes" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gbrlsnchs/jwt/v3" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/internal/database" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/router/tokens" + "github.com/Rene-Roscher/wings/server" +) + +const uploadTestServerUuid = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee" + +// countingBody counts the bytes a handler reads from a request body. +type countingBody struct { + io.ReadCloser + read *atomic.Int64 +} + +func (b countingBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + b.read.Add(int64(n)) + return n, err +} + +var ( + testDatabaseOnce sync.Once + testDatabaseErr error +) + +// initTestDatabase opens the activity database, which can only be opened once +// per process. +func initTestDatabase(t *testing.T) { + t.Helper() + testDatabaseOnce.Do(func() { + dir, err := os.MkdirTemp("", "wings-router-test") + if err != nil { + testDatabaseErr = err + return + } + cfg := config.Get() + cfg.System.RootDirectory = dir + config.Set(cfg) + testDatabaseErr = database.Initialize() + }) + require.NoError(t, testDatabaseErr) +} + +type uploadTest struct { + url string + root string + read *atomic.Int64 +} + +// newUploadTest starts the router with one server whose files live in a +// temporary directory. uploadLimit is in MiB and diskLimit in MiB (0 is +// unlimited). +func newUploadTest(t *testing.T, uploadLimit int64, diskLimit int64) *uploadTest { + t.Helper() + previous := config.Get() + t.Cleanup(func() { config.Set(previous) }) + + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.PanelLocation = "http://panel.test" + cfg.System.Data = t.TempDir() + cfg.System.User.Uid = os.Getuid() + cfg.System.User.Gid = os.Getgid() + cfg.Api.UploadLimit = uploadLimit + config.Set(cfg) + initTestDatabase(t) + + settings, err := json.Marshal(map[string]any{ + "uuid": uploadTestServerUuid, + "build": map[string]any{"disk_space": diskLimit}, + }) + require.NoError(t, err) + manager := server.NewEmptyManager(nil) + s, err := manager.InitServer(uploadTestServerUuid, remote.ServerConfigurationResponse{Settings: settings, ProcessConfiguration: &remote.ProcessConfiguration{}}) + require.NoError(t, err) + t.Cleanup(s.CtxCancel) + manager.Add(s) + + read := &atomic.Int64{} + engine := Configure(manager, nil) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Body = countingBody{ReadCloser: r.Body, read: read} + engine.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + return &uploadTest{url: srv.URL, root: s.Filesystem().Path(), read: read} +} + +func (u *uploadTest) token(t *testing.T) string { + t.Helper() + // Tokens issued before Wings boots are rejected, and the issued time only has + // second precision. + now := time.Now().Add(time.Second * 2) + payload := tokens.UploadPayload{ + Payload: jwt.Payload{ + IssuedAt: jwt.NumericDate(now), + ExpirationTime: jwt.NumericDate(now.Add(time.Minute * 15)), + }, + Scoped: tokens.Scoped{Scope: string(tokens.FileUpload)}, + ServerUuid: uploadTestServerUuid, + UserUuid: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + UniqueId: uuid.NewString(), + } + token, err := jwt.Sign(&payload, config.GetJwtAlgorithm()) + require.NoError(t, err) + return string(token) +} + +// upload posts the files, keyed by name, and returns the response status and +// the size of the request body. +func (u *uploadTest) upload(t *testing.T, files map[string][]byte) (int, int64) { + t.Helper() + var body bytes.Buffer + mw := multipart.NewWriter(&body) + for name, content := range files { + part, err := mw.CreateFormFile("files", name) + require.NoError(t, err) + _, err = part.Write(content) + require.NoError(t, err) + } + require.NoError(t, mw.Close()) + size := int64(body.Len()) + + req, err := http.NewRequest(http.MethodPost, u.url+"/upload/file?directory=/uploads&token="+u.token(t), &body) + require.NoError(t, err) + req.Header.Set("Content-Type", mw.FormDataContentType()) + res, err := http.DefaultClient.Do(req) + if err != nil { + // The server may close the connection before reading all of the body. + return 0, size + } + _ = res.Body.Close() + return res.StatusCode, size +} + +func TestUploadWritesFiles(t *testing.T) { + u := newUploadTest(t, 1, 0) + + status, _ := u.upload(t, map[string][]byte{ + "a.txt": []byte("first"), + "b.txt": []byte("second"), + }) + require.Equal(t, http.StatusOK, status) + + for name, want := range map[string]string{"a.txt": "first", "b.txt": "second"} { + got, err := os.ReadFile(filepath.Join(u.root, "uploads", name)) + require.NoError(t, err) + require.Equal(t, want, string(got)) + } +} + +// A file over the upload limit is refused once the limit is passed. +func TestUploadRejectsFileOverLimit(t *testing.T) { + u := newUploadTest(t, 1, 0) + + status, size := u.upload(t, map[string][]byte{"big.bin": bytes.Repeat([]byte("a"), 40<<20)}) + if status != 0 { + require.Equal(t, http.StatusBadRequest, status) + } + require.Less(t, u.read.Load(), size/2, "read %d of %d request body bytes before refusing the file", u.read.Load(), size) + + _, err := os.Stat(filepath.Join(u.root, "uploads", "big.bin")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +// Uploads count against the server's disk limit while they are written. +func TestUploadRespectsDiskLimit(t *testing.T) { + u := newUploadTest(t, 10, 1) + + status, _ := u.upload(t, map[string][]byte{"big.bin": bytes.Repeat([]byte("a"), 2<<20)}) + require.GreaterOrEqual(t, status, http.StatusBadRequest) + + _, err := os.Stat(filepath.Join(u.root, "uploads", "big.bin")) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/router/router_server_restore_test.go b/router/router_server_restore_test.go new file mode 100644 index 000000000..688ce2683 --- /dev/null +++ b/router/router_server_restore_test.go @@ -0,0 +1,61 @@ +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apex/log" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/remote" + wserver "github.com/Rene-Roscher/wings/server" +) + +// A restore of a backup that cannot be found does not delete the server's +// files, even when truncate_directory is set. +func TestRestoreOfMissingBackupKeepsFiles(t *testing.T) { + previous := config.Get() + t.Cleanup(func() { config.Set(previous) }) + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.System.Data = t.TempDir() + cfg.System.BackupDirectory = t.TempDir() + config.Set(cfg) + + const uuid = "dddddddd-dddd-4ddd-8ddd-dddddddddddd" + settings, err := json.Marshal(map[string]any{"uuid": uuid}) + require.NoError(t, err) + client := backupTestRemoteClient{} + s, err := wserver.NewEmptyManager(client).InitServer(uuid, remote.ServerConfigurationResponse{Settings: settings, ProcessConfiguration: &remote.ProcessConfiguration{}}) + require.NoError(t, err) + t.Cleanup(s.CtxCancel) + s.Environment = backupTestEnvironment{} + keep := filepath.Join(s.Filesystem().Path(), "keep.txt") + require.NoError(t, os.WriteFile(keep, []byte("important"), 0o644)) + + gin.SetMode(gin.TestMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + backupID := "6a3c2f6e-1b7d-4c8e-9f0a-2d4b6c8e0f12" + c.Request = httptest.NewRequest(http.MethodPost, "/api/servers/"+uuid+"/backup/"+backupID+"/restore", strings.NewReader(`{"adapter":"wings","truncate_directory":true}`)) + c.Request.Header.Set("Content-Type", "application/json") + c.Params = gin.Params{{Key: "server", Value: uuid}, {Key: "backup", Value: backupID}} + c.Set("server", s) + c.Set("api_client", client) + c.Set("logger", log.WithField("test", t.Name())) + + postServerRestoreBackup(c) + + require.NotEmpty(t, c.Errors, "expected the restore to fail") + _, err = os.Stat(keep) + require.NoError(t, err, "the server's files were deleted although the backup could not be found") + require.False(t, s.IsRestoring()) +} diff --git a/router/router_server_transfer.go b/router/router_server_transfer.go index 758d33a05..c20e6b01c 100644 --- a/router/router_server_transfer.go +++ b/router/router_server_transfer.go @@ -32,15 +32,9 @@ func postServerTransfer(c *gin.Context) { s := ExtractServer(c) - // Check if the server is already being transferred or has other operations running. - // There will be another endpoint for resetting this value either by deleting the - // server, or by canceling the transfer. - if s.IsTransferring() { - c.AbortWithStatusJSON(http.StatusConflict, gin.H{ - "error": "A transfer is already in progress for this server.", - }) - return - } + // Fork: refuse to transfer while a backup or restore operation is running. + // These checks come before StartTransferring so a rejected request never + // leaves the server marked as transferring. if s.IsBackingUp() { c.AbortWithStatusJSON(http.StatusConflict, gin.H{ "error": "A backup operation is already running for this server - transfer blocked.", @@ -54,6 +48,17 @@ func postServerTransfer(c *gin.Context) { return } + // Mark the server as being transferred, which blocks it from starting. This + // fails if a transfer is already in progress, or if the server is being + // installed or restored. There will be another endpoint for resetting this + // value either by deleting the server, or by canceling the transfer. + if !s.StartTransferring() { + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "The server cannot be transferred while it is being transferred, installed, or restored.", + }) + return + } + manager := middleware.ExtractManager(c) notifyPanelOfFailure := func() { @@ -68,9 +73,6 @@ func postServerTransfer(c *gin.Context) { s.SetTransferring(false) } - // Block the server from starting while we are transferring it. - s.SetTransferring(true) - // Ensure the server is offline. Sometimes a "No such container" error gets through // which means the server is already stopped. We can ignore that. if s.Environment.State() != environment.ProcessOfflineState { @@ -87,7 +89,13 @@ func postServerTransfer(c *gin.Context) { // Create a new transfer instance for this server. trnsfr := transfer.New(context.Background(), s) - transfer.Outgoing().Add(trnsfr) + if !transfer.Outgoing().Add(trnsfr) { + s.SetTransferring(false) + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A transfer is already in progress for this server.", + }) + return + } go func() { defer transfer.Outgoing().Remove(trnsfr) diff --git a/router/router_server_ws.go b/router/router_server_ws.go index 9ae1cf1d4..da8e4a834 100644 --- a/router/router_server_ws.go +++ b/router/router_server_ws.go @@ -4,14 +4,15 @@ import ( "context" "encoding/json" "net/http" + "sync" "time" "emperror.dev/errors" - "github.com/gin-gonic/gin" - ws "github.com/gorilla/websocket" "github.com/Rene-Roscher/wings/router/middleware" "github.com/Rene-Roscher/wings/router/websocket" "github.com/Rene-Roscher/wings/server" + "github.com/gin-gonic/gin" + ws "github.com/gorilla/websocket" "golang.org/x/time/rate" ) @@ -23,14 +24,33 @@ var expectedCloseCodes = []int{ ws.CloseServiceRestart, } -// Upgrades a connection to a websocket and passes events along between. -func getServerWebsocket(c *gin.Context) { +var ( + // maxWebsocketConnections is the maximum number of authenticated websocket + // connections a server can have open at once. + maxWebsocketConnections = 30 + // websocketAuthenticationTimeout is the amount of time a websocket connection + // has to authenticate before it is closed. + websocketAuthenticationTimeout = 30 * time.Second + + errTooManyWebsockets = errors.New("too many open websocket connections") +) + +// getServerWebsocket returns a handler that upgrades a connection to a websocket +// and passes events along between, allowing at most maxConnections authenticated +// connections to a server, each of which must authenticate within authTimeout. +func getServerWebsocket(maxConnections int, authTimeout time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + handleServerWebsocket(c, maxConnections, authTimeout) + } +} + +func handleServerWebsocket(c *gin.Context, maxConnections int, authTimeout time.Duration) { manager := middleware.ExtractManager(c) s, _ := manager.Get(c.Param("server")) - // Limit the total number of websockets that can be opened at any one time for - // a server instance. This applies across all users connected to the server, and - // is not applied on a per-user basis. + // Limit the total number of authenticated websockets that can be open at any one + // time for a server instance. This applies across all users connected to the server, + // and is not applied on a per-user basis. // // todo: it would be great to make this per-user instead, but we need to modify // how we even request this endpoint in order for that to be possible. Some type @@ -38,7 +58,7 @@ func getServerWebsocket(c *gin.Context) { // panel using a shared secret is likely the easiest option. The benefit of that // is that we can both scope things to the user before authentication, and also // verify that the JWT provided by the panel is assigned to the same user. - if s.Websockets().Len() >= 30 { + if s.Websockets().Len() >= maxConnections { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Too many open websocket connections.", }) @@ -62,22 +82,47 @@ func getServerWebsocket(c *gin.Context) { return } - // Track this open connection on the server so that we can close them all programmatically - // if the server is deleted. - s.Websockets().Push(handler.Uuid(), &cancel) + // Track this connection on the server once it authenticates so that we can close them + // all programmatically when access to the server is revoked. Connections only count + // towards the limit once they have authenticated, and those that do not + // authenticate in time are closed. + // + // Messages are handled in their own goroutines which can outlive this function, so + // a connection that authenticates after it has closed is not tracked. + // Connections that do not authenticate in time are closed. + var ( + trackMu sync.Mutex + closed bool + ) + authTimer := time.AfterFunc(authTimeout, cancel) + defer authTimer.Stop() + handler.OnAuthenticated(func() error { + trackMu.Lock() + defer trackMu.Unlock() + if closed || !authTimer.Stop() { + return context.Canceled + } + if !s.Websockets().TryPush(handler.Uuid(), &cancel, handler.User, maxConnections) { + cancel() + return errTooManyWebsockets + } + return nil + }) handler.Logger().Debug("opening connection to server websocket") - defer s.Websockets().Remove(handler.Uuid()) + defer func() { + trackMu.Lock() + closed = true + trackMu.Unlock() + s.Websockets().Remove(handler.Uuid()) + }() go func() { - select { // When the main context is canceled (through disconnect, server deletion, or server // suspension) close the connection itself. - case <-ctx.Done(): - handler.Logger().Debug("closing connection to server websocket") - if err := handler.Connection.Close(); err != nil { - handler.Logger().WithError(err).Error("failed to close websocket connection") - } - break + <-ctx.Done() + handler.Logger().Debug("closing connection to server websocket") + if err := handler.Connection.Close(); err != nil { + handler.Logger().WithError(err).Error("failed to close websocket connection") } }() @@ -100,7 +145,7 @@ func getServerWebsocket(c *gin.Context) { // the HTTP response in the websocket client, thus we connect and then // immediately close with failure. if s.IsSuspended() { - _ = handler.Connection.WriteMessage(ws.CloseMessage, ws.FormatCloseMessage(4409, "server is suspended")) + _ = handler.SendClose(4409, "server is suspended") return } @@ -126,7 +171,7 @@ func getServerWebsocket(c *gin.Context) { if !rl.Allow() { if !throttled { throttled = true - _ = handler.Connection.WriteJSON(websocket.Message{Event: websocket.ThrottledEvent, Args: []string{"global"}}) + _ = handler.SendThrottled("global") } continue } @@ -150,6 +195,13 @@ func getServerWebsocket(c *gin.Context) { } go func(msg websocket.Message) { + // Close this connection if handling the message panics. + defer func() { + if r := recover(); r != nil { + handler.Logger().WithField("panic", r).Error("recovered from panic while handling websocket message") + cancel() + } + }() if err := handler.HandleInbound(ctx, msg); err != nil { if errors.Is(err, server.ErrSuspended) { cancel() diff --git a/router/router_server_ws_test.go b/router/router_server_ws_test.go new file mode 100644 index 000000000..e3cb9577b --- /dev/null +++ b/router/router_server_ws_test.go @@ -0,0 +1,231 @@ +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gbrlsnchs/jwt/v3" + ws "github.com/gorilla/websocket" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/router/tokens" + "github.com/Rene-Roscher/wings/router/websocket" + "github.com/Rene-Roscher/wings/server" +) + +const websocketTestServerUuid = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + +type runningEnvironment struct { + environment.ProcessEnvironment +} + +func (runningEnvironment) State() string { + return environment.ProcessRunningState +} + +func startWebsocketTestServer(t *testing.T) string { + t.Helper() + url, _ := startWebsocketTestServerFor(t) + return url +} + +func startWebsocketTestServerFor(t *testing.T) (string, *server.Server) { + t.Helper() + manager, s := newWebsocketTestManager(t) + srv := httptest.NewServer(Configure(manager, nil)) + t.Cleanup(srv.Close) + return "ws" + strings.TrimPrefix(srv.URL, "http") + "/api/servers/" + websocketTestServerUuid + "/ws", s +} + +// newWebsocketTestManager returns a manager holding a single running server. +func newWebsocketTestManager(t *testing.T) (*server.Manager, *server.Server) { + t.Helper() + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + cfg.PanelLocation = "http://panel.test" + config.Set(cfg) + + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + settings, _ := json.Marshal(map[string]string{"uuid": websocketTestServerUuid}) + if err := s.SyncWithConfiguration(remote.ServerConfigurationResponse{Settings: settings, ProcessConfiguration: &remote.ProcessConfiguration{}}); err != nil { + t.Fatal(err) + } + s.Environment = runningEnvironment{} + manager := server.NewEmptyManager(nil) + manager.Add(s) + return manager, s +} + +func dialWebsocket(t *testing.T, url string) (*ws.Conn, error) { + t.Helper() + conn, _, err := ws.DefaultDialer.Dial(url, http.Header{"Origin": []string{"http://panel.test"}}) + if err == nil { + t.Cleanup(func() { _ = conn.Close() }) + } + return conn, err +} + +// authenticationMessage returns a message authenticating with a valid token. +func authenticationMessage(t *testing.T) websocket.Message { + t.Helper() + // Tokens issued before Wings booted are rejected, and the issued time only has + // second precision. + now := time.Now().Add(time.Second * 2) + payload := tokens.WebsocketPayload{ + Payload: jwt.Payload{ + IssuedAt: jwt.NumericDate(now), + ExpirationTime: jwt.NumericDate(now.Add(time.Minute * 10)), + }, + Scoped: tokens.Scoped{Scope: string(tokens.Websocket)}, + UserUUID: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + ServerUUID: websocketTestServerUuid, + Permissions: []string{websocket.PermissionConnect}, + } + token, err := jwt.Sign(&payload, config.GetJwtAlgorithm()) + if err != nil { + t.Fatal(err) + } + return websocket.Message{Event: websocket.AuthenticationEvent, Args: []string{string(token)}} +} + +// authenticate sends a valid token over the socket and reports whether the +// server accepted it. +func authenticate(t *testing.T, conn *ws.Conn) bool { + t.Helper() + if err := conn.WriteJSON(authenticationMessage(t)); err != nil { + return false + } + _ = conn.SetReadDeadline(time.Now().Add(time.Second * 5)) + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + for { + var m websocket.Message + if err := conn.ReadJSON(&m); err != nil { + return false + } + switch m.Event { + case websocket.AuthenticationSuccessEvent: + return true + case websocket.ErrorEvent, websocket.JwtErrorEvent: + return false + } + } +} + +// Messages are handled in goroutines that can outlive the connection, so a client +// that authenticates and disconnects straight away must not leave its slot behind. +func TestWebsocketSlotsAreReleasedWhenClosedWhileAuthenticating(t *testing.T) { + url, s := startWebsocketTestServerFor(t) + + msg := authenticationMessage(t) + for i := 0; i < 200; i++ { + conn, err := dialWebsocket(t, url) + if err != nil { + t.Fatalf("unexpected error opening connection %d: %v", i, err) + } + _ = conn.WriteJSON(msg) + _ = conn.UnderlyingConn().Close() + } + + deadline := time.Now().Add(time.Second * 5) + for s.Websockets().Len() > 0 { + if time.Now().After(deadline) { + t.Fatalf("expected every slot to be released, %d are still held", s.Websockets().Len()) + } + time.Sleep(time.Millisecond * 50) + } + + conn, err := dialWebsocket(t, url) + if err != nil { + t.Fatalf("expected a user to be able to connect, got %v", err) + } + if !authenticate(t, conn) { + t.Fatal("expected a user to be able to authenticate") + } +} + +func TestUnauthenticatedWebsocketsDoNotUseConnectionSlots(t *testing.T) { + url := startWebsocketTestServer(t) + + // Connections that never authenticate. + for i := 0; i < 30; i++ { + if _, err := dialWebsocket(t, url); err != nil { + t.Fatalf("unexpected error opening connection %d: %v", i, err) + } + } + + conn, err := dialWebsocket(t, url) + if err != nil { + t.Fatalf("expected a user to be able to connect, got %v", err) + } + if !authenticate(t, conn) { + t.Fatal("expected a user to be able to authenticate") + } +} + +func TestUnauthenticatedWebsocketsAreClosed(t *testing.T) { + previous := websocketAuthenticationTimeout + websocketAuthenticationTimeout = time.Millisecond * 300 + t.Cleanup(func() { websocketAuthenticationTimeout = previous }) + url := startWebsocketTestServer(t) + + idle, err := dialWebsocket(t, url) + if err != nil { + t.Fatal(err) + } + authenticated, err := dialWebsocket(t, url) + if err != nil { + t.Fatal(err) + } + if !authenticate(t, authenticated) { + t.Fatal("expected socket to authenticate") + } + + _ = idle.SetReadDeadline(time.Now().Add(time.Second * 3)) + if _, _, err := idle.ReadMessage(); err == nil || strings.Contains(err.Error(), "i/o timeout") { + t.Fatalf("expected the server to close an unauthenticated socket, got %v", err) + } + + // The authenticated socket is not affected by the deadline. + time.Sleep(websocketAuthenticationTimeout * 2) + if !authenticate(t, authenticated) { + t.Fatal("expected the authenticated socket to remain open") + } +} + +func TestAuthenticatedWebsocketsAreLimited(t *testing.T) { + previous := maxWebsocketConnections + maxWebsocketConnections = 2 + t.Cleanup(func() { maxWebsocketConnections = previous }) + url := startWebsocketTestServer(t) + + var conns []*ws.Conn + for i := 0; i < 3; i++ { + conn, err := dialWebsocket(t, url) + if err != nil { + t.Fatal(err) + } + conns = append(conns, conn) + } + if !authenticate(t, conns[0]) || !authenticate(t, conns[1]) { + t.Fatal("expected sockets within the limit to authenticate") + } + if authenticate(t, conns[2]) { + t.Fatal("expected socket over the limit to fail to authenticate") + } + if _, err := dialWebsocket(t, url); err == nil { + t.Fatal("expected new connections to be refused once the limit is reached") + } +} diff --git a/router/router_server_ws_write_test.go b/router/router_server_ws_write_test.go new file mode 100644 index 000000000..89710ca76 --- /dev/null +++ b/router/router_server_ws_write_test.go @@ -0,0 +1,170 @@ +package router + +import ( + "bytes" + "errors" + "net" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/router/websocket" +) + +// blockingConn is a connection whose writes can be made to block until it is +// closed. +type blockingConn struct { + net.Conn + blocked atomic.Bool + closeOnce sync.Once + closed chan struct{} +} + +func (c *blockingConn) Write(p []byte) (int, error) { + if c.blocked.Load() { + <-c.closed + return 0, net.ErrClosed + } + return c.Conn.Write(p) +} + +func (c *blockingConn) Close() error { + c.closeOnce.Do(func() { close(c.closed) }) + return c.Conn.Close() +} + +type blockingListener struct { + net.Listener + accepted chan *blockingConn +} + +func (l *blockingListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + sc := &blockingConn{Conn: conn, closed: make(chan struct{})} + l.accepted <- sc + return sc, nil +} + +// syncBuffer is a bytes.Buffer that can be written to from several goroutines. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// startBlockingWebsocketServer starts the websocket test server with a listener +// whose connections can block writes. Panics recovered by the router are written +// to the returned buffer. +func startBlockingWebsocketServer(t *testing.T) (string, *blockingListener, *syncBuffer) { + t.Helper() + recovered := &syncBuffer{} + previous := gin.DefaultErrorWriter + gin.DefaultErrorWriter = recovered + t.Cleanup(func() { gin.DefaultErrorWriter = previous }) + + manager, _ := newWebsocketTestManager(t) + + base, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + listener := &blockingListener{Listener: base, accepted: make(chan *blockingConn, 4)} + srv := httptest.NewUnstartedServer(Configure(manager, nil)) + require.NoError(t, srv.Listener.Close()) + srv.Listener = listener + srv.Start() + t.Cleanup(srv.Close) + + return "ws" + strings.TrimPrefix(srv.URL, "http") + "/api/servers/" + websocketTestServerUuid + "/ws", listener, recovered +} + +func isNetTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} + +// A socket that does not authenticate is closed once the authentication timeout +// passes. +func TestUnauthenticatedSocketIsClosedAfterTimeout(t *testing.T) { + previous := websocketAuthenticationTimeout + websocketAuthenticationTimeout = 300 * time.Millisecond + t.Cleanup(func() { websocketAuthenticationTimeout = previous }) + + url, listener, _ := startBlockingWebsocketServer(t) + conn, err := dialWebsocket(t, url) + require.NoError(t, err) + serverSide := <-listener.accepted + t.Cleanup(func() { _ = serverSide.Close() }) + + // Ten messages pass the per-connection limiter. Each is answered with a JWT + // error, and the per-event limiter adds one throttling notice. + for i := 0; i < 10; i++ { + require.NoError(t, conn.WriteJSON(websocket.Message{Event: websocket.SendStatsEvent})) + } + require.NoError(t, conn.SetReadDeadline(time.Now().Add(2*time.Second))) + for i := 0; i < 5; i++ { + var m websocket.Message + require.NoError(t, conn.ReadJSON(&m), "reply %d", i) + } + + // Block writes to the client, then send one more message, which is answered + // with a throttling notice. + serverSide.blocked.Store(true) + require.NoError(t, conn.WriteJSON(websocket.Message{Event: websocket.SendStatsEvent})) + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + _, _, err = conn.ReadMessage() + require.Error(t, err) + require.False(t, isNetTimeout(err), "expected the socket to be closed after the authentication timeout: %v", err) +} + +// Only one write to a socket may be in progress at a time, including the +// throttling notice written by the read loop. +func TestWebsocketWritesAreSerialized(t *testing.T) { + previous := websocketAuthenticationTimeout + websocketAuthenticationTimeout = 500 * time.Millisecond + t.Cleanup(func() { websocketAuthenticationTimeout = previous }) + + url, listener, recovered := startBlockingWebsocketServer(t) + conn, err := dialWebsocket(t, url) + require.NoError(t, err) + serverSide := <-listener.accepted + t.Cleanup(func() { _ = serverSide.Close() }) + + // Block the reply to the first message while it is being written. + serverSide.blocked.Store(true) + require.NoError(t, conn.WriteJSON(websocket.Message{Event: websocket.SendStatsEvent})) + time.Sleep(100 * time.Millisecond) + + // Send more messages than the per-connection limiter allows while that write + // is in progress. + for i := 0; i < 12; i++ { + require.NoError(t, conn.WriteJSON(websocket.Message{Event: websocket.SendStatsEvent})) + } + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(3*time.Second))) + _, _, err = conn.ReadMessage() + require.Error(t, err) + // The connection can close before a recovered panic is logged. + require.Never(t, func() bool { + return strings.Contains(recovered.String(), "concurrent write") + }, 500*time.Millisecond, 10*time.Millisecond, "expected writes to the socket not to overlap") +} diff --git a/router/router_system.go b/router/router_system.go index d51c5ea0c..54a044050 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -4,11 +4,10 @@ import ( "context" "errors" "net/http" - "strings" + "github.com/Rene-Roscher/wings/router/tokens" "github.com/apex/log" "github.com/gin-gonic/gin" - "github.com/Rene-Roscher/wings/router/tokens" "github.com/Rene-Roscher/wings/config" "github.com/Rene-Roscher/wings/router/middleware" @@ -49,7 +48,7 @@ func getSystemInformation(c *gin.Context) { // this wings instance. func getAllServers(c *gin.Context) { servers := middleware.ExtractManager(c).All() - out := make([]server.APIResponse, len(servers), len(servers)) + out := make([]server.APIResponse, len(servers)) for i, v := range servers { out[i] = v.ToAPIResponse() } @@ -66,6 +65,14 @@ func postCreateServer(c *gin.Context) { return } + // A server can only exist once on an instance. + if _, ok := manager.Get(details.UUID); ok { + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A server with this identifier already exists on this instance.", + }) + return + } + install, err := installer.New(c.Request.Context(), manager, details) if err != nil { if installer.IsValidationError(err) { @@ -81,7 +88,13 @@ func postCreateServer(c *gin.Context) { // Plop that server instance onto the request so that it can be referenced in // requests from here-on out. - manager.Add(install.Server()) + if !manager.AddIfMissing(install.Server()) { + install.Server().CleanupForDestroy() + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A server with this identifier already exists on this instance.", + }) + return + } // Begin the installation process in the background to not block the request // cycle. If there are any errors they will be logged and communicated back @@ -129,19 +142,17 @@ func postUpdateConfiguration(c *gin.Context) { return } - if err := c.BindJSON(&cfg); err != nil { + // Values missing from the request keep their current value. + update := cfg.PanelConfiguration() + if err := c.BindJSON(&update); err != nil { return } - // Keep the SSL certificates the same since the Panel will send through Lets Encrypt - // default locations. However, if we picked a different location manually we don't - // want to override that. - // - // If you pass through manual locations in the API call this logic will be skipped. - if strings.HasPrefix(cfg.Api.Ssl.KeyFile, "/etc/letsencrypt/live/") { - cfg.Api.Ssl.KeyFile = config.Get().Api.Ssl.KeyFile - cfg.Api.Ssl.CertificateFile = config.Get().Api.Ssl.CertificateFile - } + // Keep the SSL certificates the same since the Panel always sends the default Lets + // Encrypt locations, which would override any location picked manually. + update.Api.Ssl.CertificateFile = cfg.Api.Ssl.CertificateFile + update.Api.Ssl.KeyFile = cfg.Api.Ssl.KeyFile + cfg.ApplyPanelConfiguration(update) // The token that everything authenticates against is a derived value that is // not part of the payload sent by the Panel, so it has to be re-resolved from @@ -194,14 +205,14 @@ func postDeauthorizeUser(c *gin.Context) { for _, uuid := range data.Servers { if s, ok := m.Get(uuid); ok { tokens.DenyForServer(s.ID(), data.User) - s.Websockets().CancelAll() + s.Websockets().CancelUser(data.User) s.Sftp().Cancel(data.User) } } } else { for _, s := range m.All() { tokens.DenyForServer(s.ID(), data.User) - s.Websockets().CancelAll() + s.Websockets().CancelUser(data.User) s.Sftp().Cancel(data.User) } } diff --git a/router/router_system_test.go b/router/router_system_test.go index 995f1caf1..44b5e7c9f 100644 --- a/router/router_system_test.go +++ b/router/router_system_test.go @@ -1,14 +1,19 @@ package router import ( + "encoding/json" + "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" "github.com/gin-gonic/gin" + "gopkg.in/yaml.v2" "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/remote" "github.com/Rene-Roscher/wings/server" ) @@ -53,3 +58,129 @@ func TestPostUpdateConfigurationRotatesCredentials(t *testing.T) { t.Fatal("expected client credentials to be rotated") } } + +func TestPostCreateServerRejectsExistingServer(t *testing.T) { + const uuid = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + settings, _ := json.Marshal(map[string]string{"uuid": uuid}) + if err := s.SyncWithConfiguration(remote.ServerConfigurationResponse{Settings: settings, ProcessConfiguration: &remote.ProcessConfiguration{}}); err != nil { + t.Fatal(err) + } + manager := server.NewEmptyManager(backupTestRemoteClient{}) + manager.Add(s) + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Set("manager", manager) + c.Request = httptest.NewRequest("POST", "/api/servers", strings.NewReader(`{"uuid":"`+uuid+`"}`)) + c.Request.Header.Set("Content-Type", "application/json") + + postCreateServer(c) + + if recorder.Code != http.StatusConflict { + t.Fatalf("expected a conflict, got status %d", recorder.Code) + } + if manager.Len() != 1 { + t.Fatalf("expected one server, got %d", manager.Len()) + } +} + +func TestPostUpdateConfigurationOnlyAppliesPanelValues(t *testing.T) { + t.Setenv("WINGS_TOKEN_ID", "") + t.Setenv("WINGS_TOKEN", "") + + p := filepath.Join(t.TempDir(), "config.yml") + cfg, err := config.NewAtPath(p) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationTokenId = "old-id" + cfg.AuthenticationToken = "old-token" + cfg.System.Username = "pterodactyl" + cfg.System.User.Uid = 988 + cfg.System.User.Gid = 988 + if err := cfg.ResolveToken(false); err != nil { + t.Fatal(err) + } + config.Set(cfg) + + // Everything the Panel sends, plus values that only the node operator may set. + body := `{ + "debug": true, + "uuid": "node-uuid", + "token_id": "new-id", + "token": "new-token", + "api": { + "host": "127.0.0.1", + "port": 8443, + "ssl": {"enabled": true, "cert": "/srv/certs/node.pem", "key": "/srv/certs/node.key"}, + "upload_limit": 250, + "trusted_proxies": ["10.0.0.0/8"] + }, + "system": { + "data": "/srv/data", + "root_directory": "/srv/root", + "sftp": {"bind_port": 2023, "bind_address": "127.0.0.2", "read_only": true}, + "user": {"uid": 1000, "gid": 1000, "rootless": {"enabled": true}}, + "username": "other", + "passwd": {"enabled": true, "directory": "/srv/passwd"}, + "machineid": {"enabled": true, "directory": "/srv/machine-id"}, + "openatmode": "openat", + "backups": {"restore_host_allowlist": ["10.0.0.0/8"]} + }, + "docker": { + "network": {"mode": "bridge"}, + "userns_mode": "private", + "registries": {"registry.example.com": {"username": "u", "password": "p"}} + }, + "allowed_mounts": ["/srv"], + "allowed_origins": ["https://other.invalid"], + "remote": "http://other.invalid", + "ignore_panel_config_updates": false + }` + + expected := config.Get() + expected.Debug = true + expected.Uuid = "node-uuid" + expected.AuthenticationTokenId = "new-id" + expected.AuthenticationToken = "new-token" + expected.Api.Host = "127.0.0.1" + expected.Api.Port = 8443 + expected.Api.Ssl.Enabled = true + expected.Api.UploadLimit = 250 + expected.System.Sftp.Port = 2023 + want, err := yaml.Marshal(expected) + if err != nil { + t.Fatal(err) + } + + manager := server.NewEmptyManager(backupTestRemoteClient{credentials: make(chan [2]string, 1)}) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Set("manager", manager) + c.Request = httptest.NewRequest("POST", "/api/update", strings.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + postUpdateConfiguration(c) + + if recorder.Code != 200 { + t.Fatalf("expected successful update, got status %d", recorder.Code) + } + got, err := yaml.Marshal(config.Get()) + if err != nil { + t.Fatal(err) + } + if string(got) != string(want) { + t.Fatalf("unexpected configuration after update:\n%s\nexpected:\n%s", got, want) + } + onDisk, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(onDisk) != string(want) { + t.Fatalf("unexpected configuration written to disk:\n%s", onDisk) + } +} diff --git a/router/router_timeout_test.go b/router/router_timeout_test.go new file mode 100644 index 000000000..4ee3c82f7 --- /dev/null +++ b/router/router_timeout_test.go @@ -0,0 +1,207 @@ +package router + +import ( + "bufio" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/server" +) + +func startTimeoutTestServer(t *testing.T) string { + t.Helper() + addr, _ := startTimeoutTestServerWithClosed(t) + return addr +} + +// startTimeoutTestServerWithClosed starts the server as it is configured for +// Wings, with shorter timeouts, also returning the number of connections the +// server has closed. +func startTimeoutTestServerWithClosed(t *testing.T) (string, *atomic.Int64) { + t.Helper() + t.Setenv("WINGS_TOKEN_ID", "") + t.Setenv("WINGS_TOKEN", "") + + prh, pit, pur := ReadHeaderTimeout, IdleTimeout, unauthenticatedTimeout + ReadHeaderTimeout, IdleTimeout, unauthenticatedTimeout = time.Millisecond*300, time.Millisecond*300, time.Millisecond*500 + t.Cleanup(func() { + ReadHeaderTimeout, IdleTimeout, unauthenticatedTimeout = prh, pit, pur + }) + + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationTokenId = "test-id" + cfg.AuthenticationToken = "test-token" + if err := cfg.ResolveToken(false); err != nil { + t.Fatal(err) + } + config.Set(cfg) + + manager := server.NewEmptyManager(backupTestRemoteClient{credentials: make(chan [2]string, 1)}) + srv := httptest.NewUnstartedServer(nil) + srv.Config = NewServer("", Configure(manager, nil), nil) + var closed atomic.Int64 + srv.Config.ConnState = func(_ net.Conn, st http.ConnState) { + if st == http.StateClosed { + closed.Add(1) + } + } + srv.Start() + t.Cleanup(srv.Close) + + return strings.TrimPrefix(srv.URL, "http://"), &closed +} + +// Requests have a deadline whichever way they are routed. +func TestSlowRequestBodiesAreTimedOutBeforeRouting(t *testing.T) { + requests := map[string]string{ + "redirect": "POST /upload/file/ HTTP/1.1\r\nHost: wings\r\nContent-Length: 1000\r\n\r\n", + "chunked": "POST /api/servers/ HTTP/1.1\r\nHost: wings\r\nTransfer-Encoding: chunked\r\n\r\n", + "options": "OPTIONS * HTTP/1.1\r\nHost: wings\r\nContent-Length: 1000\r\n\r\n", + } + for name, head := range requests { + t.Run(name, func(t *testing.T) { + addr := startTimeoutTestServer(t) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if _, err := conn.Write([]byte(head)); err != nil { + t.Fatal(err) + } + body := strings.Repeat("a", 1000) + if strings.Contains(head, "chunked") { + body = strings.Repeat("1\r\na\r\n", 200) + } + go trickle(conn, body, time.Millisecond*50) + if err := waitForClose(conn, time.Second*3); err != nil { + t.Fatal(err) + } + }) + } +} + +// Connections whose responses are not read are closed. +func TestSlowResponseReadsAreTimedOut(t *testing.T) { + addr, closed := startTimeoutTestServerWithClosed(t) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.(*net.TCPConn).SetReadBuffer(4096) + + // Enough responses to fill the socket buffers, which are never read. + go func() { + req := []byte("GET /api/system HTTP/1.1\r\nHost: wings\r\n\r\n") + for i := 0; i < 40000; i++ { + if _, err := conn.Write(req); err != nil { + return + } + } + }() + + deadline := time.Now().Add(time.Second * 5) + for closed.Load() == 0 { + if time.Now().After(deadline) { + t.Fatal("connection was not closed by the server") + } + time.Sleep(time.Millisecond * 50) + } +} + +// trickle writes the data to the connection one byte at a time. +func trickle(conn net.Conn, data string, interval time.Duration) { + for i := 0; i < len(data); i++ { + if _, err := conn.Write([]byte{data[i]}); err != nil { + return + } + time.Sleep(interval) + } +} + +// waitForClose reads from the connection until the server closes it, returning +// an error if that does not happen in time. +func waitForClose(conn net.Conn, timeout time.Duration) error { + _ = conn.SetReadDeadline(time.Now().Add(timeout)) + _, err := io.Copy(io.Discard, conn) + if errors.Is(err, os.ErrDeadlineExceeded) { + return errors.New("connection was not closed by the server") + } + return nil +} + +func TestSlowRequestHeadersAreTimedOut(t *testing.T) { + addr := startTimeoutTestServer(t) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + go trickle(conn, "GET /api/system HTTP/1.1\r\nHost: wings\r\nX-Slow: "+strings.Repeat("a", 1000), time.Millisecond*50) + if err := waitForClose(conn, time.Second*3); err != nil { + t.Fatal(err) + } +} + +func TestSlowUnauthenticatedRequestBodyIsTimedOut(t *testing.T) { + addr := startTimeoutTestServer(t) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if _, err := conn.Write([]byte("POST /api/update HTTP/1.1\r\nHost: wings\r\nContent-Type: application/json\r\nContent-Length: 1000\r\n\r\n")); err != nil { + t.Fatal(err) + } + go trickle(conn, strings.Repeat(" ", 1000), time.Millisecond*50) + if err := waitForClose(conn, time.Second*3); err != nil { + t.Fatal(err) + } +} + +func TestSlowAuthenticatedRequestBodyIsAllowed(t *testing.T) { + addr := startTimeoutTestServer(t) + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + body := `{"token_id":"test-id","token":"test-token"}` + if _, err := conn.Write([]byte("POST /api/update HTTP/1.1\r\nHost: wings\r\nAuthorization: Bearer test-token\r\nContent-Type: application/json\r\nContent-Length: " + strconv.Itoa(len(body)) + "\r\n\r\n")); err != nil { + t.Fatal(err) + } + // Takes well over the deadline for unauthenticated requests to send. + start := time.Now() + trickle(conn, body, time.Millisecond*30) + if time.Since(start) < unauthenticatedTimeout*2 { + t.Fatal("expected the body to take longer than the unauthenticated read timeout to send") + } + + _ = conn.SetReadDeadline(time.Now().Add(time.Second * 3)) + res, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != http.StatusOK { + t.Fatalf("expected authenticated request to succeed, got status %d", res.StatusCode) + } +} diff --git a/router/router_transfer.go b/router/router_transfer.go index b7b221221..e638aa8ac 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -46,6 +46,7 @@ func postTransfers(c *gin.Context) { c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden."}) return } + middleware.ClearDeadlines(c) manager := middleware.ExtractManager(c) u, err := uuid.Parse(token.Subject) @@ -59,8 +60,24 @@ func postTransfers(c *gin.Context) { ctx context.Context cancel context.CancelFunc ) - trnsfr := transfer.Incoming().Get(u.String()) - if trnsfr == nil { + // Only one request can send the server at a time. + if transfer.Incoming().Get(u.String()) != nil { + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A transfer is already in progress for this server.", + }) + return + } + + var trnsfr *transfer.Transfer + { + // A server that already exists on this instance cannot be transferred to it. + if _, ok := manager.Get(u.String()); ok { + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A server with this identifier already exists on this instance.", + }) + return + } + // TODO: should this use the request context? trnsfr = transfer.New(c, nil) @@ -72,59 +89,41 @@ func postTransfers(c *gin.Context) { StartOnCompletion: false, }) if err != nil { - if err := manager.Client().SetTransferStatus(context.Background(), trnsfr.Server.ID(), false); err != nil { - trnsfr.Log().WithField("status", false).WithError(err).Error("failed to set transfer status") + if err := manager.Client().SetTransferStatus(context.Background(), u.String(), false); err != nil { + log.WithField("server", u.String()).WithField("status", false).WithError(err).Error("failed to set transfer status") } middleware.CaptureAndAbort(c, err) return } i.Server().SetTransferring(true) - manager.Add(i.Server()) + if !manager.AddIfMissing(i.Server()) { + i.Server().CleanupForDestroy() + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A server with this identifier already exists on this instance.", + }) + return + } // We add the transfer to the list of transfers once we have a server instance to use. trnsfr.Server = i.Server() - transfer.Incoming().Add(trnsfr) - } else { - ctx, cancel = context.WithCancel(trnsfr.Context()) - defer cancel() + if !transfer.Incoming().Add(trnsfr) { + manager.Remove(func(match *server.Server) bool { return match == i.Server() }) + i.Server().CleanupForDestroy() + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "A transfer is already in progress for this server.", + }) + return + } } // Any errors past this point (until the transfer is complete) will abort // the transfer. successful := false - defer func(ctx context.Context, trnsfr *transfer.Transfer) { - // Remove the transfer from the list of incoming transfers. - transfer.Incoming().Remove(trnsfr) - - if !successful { - trnsfr.Server.Events().Publish(server.TransferStatusEvent, "failure") - manager.Remove(func(match *server.Server) bool { - return match.ID() == trnsfr.Server.ID() - }) - } - - if err := manager.Client().SetTransferStatus(context.Background(), trnsfr.Server.ID(), successful); err != nil { - // Only delete the files if the transfer actually failed, otherwise we could have - // unrecoverable data-loss. - if !successful && err != nil { - // Delete all extracted files. - go func(trnsfr *transfer.Transfer) { - _ = trnsfr.Server.Filesystem().UnixFS().Close() - if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil && !os.IsNotExist(err) { - trnsfr.Log().WithError(err).Warn("failed to delete local server files") - } - }(trnsfr) - } - - trnsfr.Log().WithField("status", successful).WithError(err).Error("failed to set transfer status on panel") - return - } - - trnsfr.Server.SetTransferring(false) - trnsfr.Server.Events().Publish(server.TransferStatusEvent, "success") - }(ctx, trnsfr) + defer func() { + completeIncomingTransfer(manager, trnsfr, successful) + }() mediaType, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) if err != nil { @@ -177,7 +176,7 @@ out: } tee := io.TeeReader(p, h) - if err := trnsfr.Server.Filesystem().ExtractStreamUnsafe(ctx, "/", tee); err != nil { + if err := trnsfr.Server.Filesystem().ExtractTransfer(ctx, tee); err != nil { middleware.CaptureAndAbort(c, err) return } @@ -193,7 +192,8 @@ out: hasChecksum = true - v, err := io.ReadAll(p) + // A SHA-256 checksum is 64 hexadecimal characters. + v, err := io.ReadAll(io.LimitReader(p, 128)) if err != nil { middleware.CaptureAndAbort(c, err) return @@ -254,18 +254,41 @@ out: trnsfr.Log().Debug("done!") } -// deleteTransfer cancels an incoming transfer for a server. -func deleteTransfer(c *gin.Context) { - s := ExtractServer(c) - - if !s.IsTransferring() { - c.AbortWithStatusJSON(http.StatusConflict, gin.H{ - "error": "Server is not currently being transferred.", +// completeIncomingTransfer finishes receiving a transfer, telling the Panel +// whether it was successful. If it was not, everything that was received is +// removed, since the server still exists on the node it was being transferred +// from. +func completeIncomingTransfer(manager *server.Manager, trnsfr *transfer.Transfer, successful bool) { + // Remove the transfer from the list of incoming transfers. + transfer.Incoming().Remove(trnsfr) + + if !successful { + trnsfr.Server.Events().Publish(server.TransferStatusEvent, "failure") + manager.Remove(func(match *server.Server) bool { + return match == trnsfr.Server }) - return + trnsfr.Server.CleanupForDestroy() + if err := trnsfr.Server.Environment.Destroy(); err != nil { + trnsfr.Log().WithError(err).Warn("failed to remove server environment") + } + if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil && !os.IsNotExist(err) { + trnsfr.Log().WithError(err).Warn("failed to delete local server files") + } + } + + if err := manager.Client().SetTransferStatus(context.Background(), trnsfr.Server.ID(), successful); err != nil { + trnsfr.Log().WithField("status", successful).WithError(err).Error("failed to set transfer status on panel") } - trnsfr := transfer.Incoming().Get(s.ID()) + if successful { + trnsfr.Server.SetTransferring(false) + trnsfr.Server.Events().Publish(server.TransferStatusEvent, "success") + } +} + +// deleteTransfer cancels an incoming transfer for a server. +func deleteTransfer(c *gin.Context) { + trnsfr := transfer.Incoming().Get(c.Param("server")) if trnsfr == nil { c.AbortWithStatusJSON(http.StatusConflict, gin.H{ "error": "Server is not currently being transferred.", diff --git a/router/router_transfer_test.go b/router/router_transfer_test.go new file mode 100644 index 000000000..8ffc6d917 --- /dev/null +++ b/router/router_transfer_test.go @@ -0,0 +1,285 @@ +package router + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gbrlsnchs/jwt/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/router/tokens" + wserver "github.com/Rene-Roscher/wings/server" + "github.com/Rene-Roscher/wings/server/transfer" +) + +// transferTestUUID is a lowercase version 4 UUID, as the installer requires. +const transferTestUUID = "3f0c9a52-7d1e-4b8a-9c2f-5e6d7a8b9c01" + +// transferTestPanel answers the Panel calls made by the transfer handlers. Only +// the calls these tests reach are implemented; any other call panics on the nil +// embedded interface. +type transferTestPanel struct { + remote.Client + + mu sync.Mutex + config remote.ServerConfigurationResponse + statusErr error + statuses []bool +} + +func (p *transferTestPanel) GetServerConfiguration(context.Context, string) (remote.ServerConfigurationResponse, error) { + return p.config, nil +} + +func (p *transferTestPanel) SetTransferStatus(_ context.Context, _ string, successful bool) error { + p.mu.Lock() + defer p.mu.Unlock() + p.statuses = append(p.statuses, successful) + return p.statusErr +} + +// transferTestEnvironment stands in for the Docker environment so that no +// container is needed. +type transferTestEnvironment struct { + environment.ProcessEnvironment +} + +func (transferTestEnvironment) Create() error { return nil } + +func (transferTestEnvironment) State() string { return environment.ProcessOfflineState } + +func (transferTestEnvironment) Destroy() error { return nil } + +type transferHarness struct { + t *testing.T + router http.Handler + mgr *wserver.Manager + panel *transferTestPanel +} + +// newTransferHarness builds the full router with a temporary data directory and +// a node token of "test-token". +func newTransferHarness(t *testing.T, panel *transferTestPanel) *transferHarness { + t.Helper() + + root := t.TempDir() + cfg, err := config.NewAtPath(filepath.Join(root, "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.System.Data = filepath.Join(root, "volumes") + cfg.System.MachineID.Enable = false + config.Set(cfg) + + settings, err := json.Marshal(map[string]string{"uuid": transferTestUUID}) + require.NoError(t, err) + panel.config = remote.ServerConfigurationResponse{ + Settings: settings, + ProcessConfiguration: &remote.ProcessConfiguration{}, + } + + mgr := wserver.NewEmptyManager(panel) + return &transferHarness{t: t, router: Configure(mgr, panel), mgr: mgr, panel: panel} +} + +// seedServer registers the test server on the manager, as an earlier receive +// would have, and replaces its Docker environment. +func (h *transferHarness) seedServer() *wserver.Server { + h.t.Helper() + + s, err := h.mgr.InitServer(transferTestUUID, h.panel.config) + require.NoError(h.t, err) + s.Environment = transferTestEnvironment{} + require.True(h.t, h.mgr.AddIfMissing(s)) + return s +} + +// startInFlightReceive marks the server as receiving a transfer, the way the +// first POST /api/transfers request does. +func (h *transferHarness) startInFlightReceive(s *wserver.Server) *transfer.Transfer { + h.t.Helper() + + s.SetTransferring(true) + tr := transfer.New(context.Background(), s) + require.True(h.t, transfer.Incoming().Add(tr)) + h.t.Cleanup(func() { transfer.Incoming().Remove(tr) }) + return tr +} + +func (h *transferHarness) postTransfer(body *bytes.Buffer, contentType, bearer string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/api/transfers", body) + req.Header.Set("Content-Type", contentType) + req.Header.Set("Authorization", "Bearer "+bearer) + w := httptest.NewRecorder() + h.router.ServeHTTP(w, req) + return w +} + +func buildTransferTestArchive(t *testing.T, files map[string]string) []byte { + t.Helper() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + })) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + return buf.Bytes() +} + +func transferTestMultipart(t *testing.T, archive []byte, checksum string) (*bytes.Buffer, string) { + t.Helper() + + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + part, err := mw.CreateFormFile("archive", "archive.tar.gz") + require.NoError(t, err) + _, err = part.Write(archive) + require.NoError(t, err) + require.NoError(t, mw.WriteField("checksum", checksum)) + require.NoError(t, mw.Close()) + return &buf, mw.FormDataContentType() +} + +// signTransferTestToken issues a transfer JWT with the same claims the Panel +// sets: subject is the server UUID, scope is "transfer", and it expires after +// 15 minutes. +func signTransferTestToken(t *testing.T, subject string) string { + t.Helper() + + token, err := jwt.Sign(tokens.TransferPayload{ + Payload: jwt.Payload{ + Subject: subject, + ExpirationTime: jwt.NumericDate(time.Now().Add(15 * time.Minute)), + }, + Scoped: tokens.Scoped{Scope: string(tokens.ServerTransfer)}, + }, config.GetJwtAlgorithm()) + require.NoError(t, err) + return string(token) +} + +// A receive whose checksum does not match does not leave the extracted files on +// disk. +func TestTransferChecksumMismatchRemovesExtractedFiles(t *testing.T) { + h := newTransferHarness(t, &transferTestPanel{}) + + archive := buildTransferTestArchive(t, map[string]string{"received.txt": "data"}) + body, contentType := transferTestMultipart(t, archive, strings.Repeat("0", 64)) + + w := h.postTransfer(body, contentType, signTransferTestToken(t, transferTestUUID)) + require.NotEqual(t, http.StatusOK, w.Code) + + _, err := os.Stat(filepath.Join(config.Get().System.Data, transferTestUUID, "received.txt")) + assert.True(t, os.IsNotExist(err), "a file from a failed receive is still on disk") + _, registered := h.mgr.Get(transferTestUUID) + assert.False(t, registered) +} + +// A second request to send a server that is already being received is refused, +// and does not affect the transfer in progress. +func TestTransferThatIsAlreadyInProgressIsRefused(t *testing.T) { + panel := &transferTestPanel{} + h := newTransferHarness(t, panel) + s := h.seedServer() + h.startInFlightReceive(s) + + archive := buildTransferTestArchive(t, map[string]string{"second.dat": "data"}) + body, contentType := transferTestMultipart(t, archive, strings.Repeat("0", 64)) + + w := h.postTransfer(body, contentType, signTransferTestToken(t, transferTestUUID)) + require.Equal(t, http.StatusConflict, w.Code) + + _, registered := h.mgr.Get(transferTestUUID) + assert.True(t, registered) + assert.True(t, s.IsTransferring()) + assert.Empty(t, panel.statuses) +} + +// A successful transfer is finished even if the Panel cannot be told about it, +// so that the server is not left unable to start. +func TestCompletedTransferIsFinishedWhenPanelFails(t *testing.T) { + panel := &transferTestPanel{statusErr: errors.New("panel unavailable")} + h := newTransferHarness(t, panel) + s := h.seedServer() + tr := h.startInFlightReceive(s) + + completeIncomingTransfer(h.mgr, tr, true) + + assert.False(t, s.IsTransferring()) + assert.Nil(t, transfer.Incoming().Get(transferTestUUID)) + assert.Equal(t, []bool{true}, panel.statuses) +} + +// A failed transfer removes the server and the files that were received. +func TestFailedTransferRemovesReceivedServer(t *testing.T) { + panel := &transferTestPanel{} + h := newTransferHarness(t, panel) + s := h.seedServer() + tr := h.startInFlightReceive(s) + require.NoError(t, s.EnsureDataDirectoryExists()) + require.NoError(t, os.WriteFile(filepath.Join(s.Filesystem().Path(), "received.txt"), []byte("data"), 0o644)) + + completeIncomingTransfer(h.mgr, tr, false) + + _, registered := h.mgr.Get(transferTestUUID) + assert.False(t, registered) + _, err := os.Stat(s.Filesystem().Path()) + assert.True(t, os.IsNotExist(err)) + assert.Equal(t, []bool{false}, panel.statuses) +} + +// An outbound transfer does not start while the server is being restored. +func TestPostServerTransferRejectsRestoringServer(t *testing.T) { + h := newTransferHarness(t, &transferTestPanel{}) + s := h.seedServer() + s.SetRestoring(true) + + payload := []byte(`{"url":"http://127.0.0.1:1/api/transfers","token":"Bearer test","server":{"uuid":"` + + transferTestUUID + `","start_on_completion":false}}`) + req := httptest.NewRequest(http.MethodPost, "/api/servers/"+transferTestUUID+"/transfer", bytes.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + h.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusConflict, w.Code) + assert.False(t, s.IsTransferring()) +} + +// Cancelling an incoming transfer that does not exist is refused, rather than +// failing with a server error. +func TestDeleteTransferForUnknownServer(t *testing.T) { + h := newTransferHarness(t, &transferTestPanel{}) + + req := httptest.NewRequest(http.MethodDelete, "/api/transfers/"+transferTestUUID, nil) + req.Header.Set("Authorization", "Bearer test-token") + w := httptest.NewRecorder() + h.router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusConflict, w.Code) +} diff --git a/router/tokens/token_store.go b/router/tokens/token_store.go index 6728efd17..3651965b9 100644 --- a/router/tokens/token_store.go +++ b/router/tokens/token_store.go @@ -12,17 +12,20 @@ type TokenStore struct { cache *cache.Cache } -var _tokens *TokenStore +var ( + _tokens *TokenStore + _tokensOnce sync.Once +) // Returns the global unique token store cache. This is used to validate // one time token usage by storing any received tokens in a local memory // cache until they are ready to expire. func getTokenStore() *TokenStore { - if _tokens == nil { + _tokensOnce.Do(func() { _tokens = &TokenStore{ cache: cache.New(time.Minute*60, time.Minute*5), } - } + }) return _tokens } @@ -35,7 +38,7 @@ func (t *TokenStore) IsValidToken(token string) bool { _, exists := t.cache.Get(token) if !exists { - t.cache.Add(token, "", time.Minute*60) + t.cache.Set(token, "", time.Minute*60) } return !exists diff --git a/router/tokens/token_store_test.go b/router/tokens/token_store_test.go new file mode 100644 index 000000000..63be64468 --- /dev/null +++ b/router/tokens/token_store_test.go @@ -0,0 +1,26 @@ +package tokens + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// A token is accepted only once. +func TestTokenIsAcceptedOnce(t *testing.T) { + var accepted atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if getTokenStore().IsValidToken("token-used-at-once") { + accepted.Add(1) + } + }() + } + wg.Wait() + require.Equal(t, int32(1), accepted.Load()) +} diff --git a/router/websocket/limiter.go b/router/websocket/limiter.go index 57315a96f..201974426 100644 --- a/router/websocket/limiter.go +++ b/router/websocket/limiter.go @@ -17,21 +17,22 @@ func (h *Handler) IsThrottled(e Event) bool { l := h.limiter.For(e) h.limiter.mu.Lock() - defer h.limiter.mu.Unlock() - if l.Allow() { h.limiter.throttles[e] = false + h.limiter.mu.Unlock() return false } // If not allowed, track the throttling and send an event over the wire // if one wasn't already sent in the same throttling period. - if v, ok := h.limiter.throttles[e]; !v || !ok { - h.limiter.throttles[e] = true - h.Logger().WithField("event", e).Debug("throttling websocket due to event volume") + notify := !h.limiter.throttles[e] + h.limiter.throttles[e] = true + h.limiter.mu.Unlock() - _ = h.unsafeSendJson(&Message{Event: ThrottledEvent, Args: []string{string(e)}}) + if notify { + h.Logger().WithField("event", e).Debug("throttling websocket due to event volume") + _ = h.SendThrottled(string(e)) } return true diff --git a/router/websocket/listeners.go b/router/websocket/listeners.go index 00dc73cfb..348a6d4ac 100644 --- a/router/websocket/listeners.go +++ b/router/websocket/listeners.go @@ -66,21 +66,6 @@ func (h *Handler) listenForExpiration(ctx context.Context) { } } -var e = []string{ - server.StatsEvent, - server.StatusEvent, - server.ConsoleOutputEvent, - server.InstallOutputEvent, - server.InstallStartedEvent, - server.InstallCompletedEvent, - server.DaemonMessageEvent, - server.BackupCompletedEvent, - server.BackupProgressEvent, - server.BackupRestoreCompletedEvent, - server.TransferLogsEvent, - server.TransferStatusEvent, -} - // ListenForServerEvents will listen for different events happening on a server // and send them along to the connected websocket client. This function will // block until the context provided to it is canceled. @@ -113,7 +98,7 @@ func (h *Handler) listenForServerEvents(ctx context.Context) error { for { select { case <-ctx.Done(): - break + // Exits through the break after the select. case b := <-logOutput: sendErr := h.SendJson(Message{Event: server.ConsoleOutputEvent, Args: []string{string(b)}}) if sendErr == nil { diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index af5061226..c97637d76 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -10,12 +10,12 @@ import ( "time" "emperror.dev/errors" + "github.com/Rene-Roscher/wings/internal/models" "github.com/apex/log" "github.com/gbrlsnchs/jwt/v3" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/gorilla/websocket" - "github.com/Rene-Roscher/wings/internal/models" "github.com/Rene-Roscher/wings/system" @@ -36,16 +36,26 @@ const ( PermissionReceiveInstall = "admin.websocket.install" PermissionReceiveTransfer = "admin.websocket.transfer" PermissionReceiveBackups = "backup.read" + PermissionReceiveActivity = "activity.read" + PermissionReceiveFiles = "file.read" ) +// writeTimeout is how long a single write to a websocket may take before the +// connection is treated as broken. +const writeTimeout = 10 * time.Second + type Handler struct { sync.RWMutex `json:"-"` - Connection *websocket.Conn `json:"-"` - jwt *tokens.WebsocketPayload - server *server.Server - ra server.RequestActivity - uuid uuid.UUID - limiter *LimiterBucket + // writeMu serializes writes to the connection, which supports only one + // concurrent writer. + writeMu sync.Mutex + Connection *websocket.Conn `json:"-"` + jwt *tokens.WebsocketPayload + server *server.Server + ra server.RequestActivity + uuid uuid.UUID + limiter *LimiterBucket + onAuthenticated func() error } var ( @@ -128,6 +138,14 @@ func (h *Handler) Uuid() uuid.UUID { return h.uuid } +// OnAuthenticated sets a function that is called when the socket authenticates +// for the first time. If it returns an error the authentication fails. +func (h *Handler) OnAuthenticated(fn func() error) { + h.Lock() + defer h.Unlock() + h.onAuthenticated = fn +} + func (h *Handler) Logger() *log.Entry { return log.WithField("subsystem", "websocket"). WithField("connection", h.Uuid().String()). @@ -136,7 +154,8 @@ func (h *Handler) Logger() *log.Entry { func (h *Handler) SendJson(v Message) error { // Do not send JSON down the line if the JWT on the connection is not valid! - if err := h.TokenValid(); err != nil { + j, err := h.validJwt() + if err != nil { _ = h.unsafeSendJson(Message{ Event: JwtErrorEvent, Args: []string{err.Error()}, @@ -144,30 +163,39 @@ func (h *Handler) SendJson(v Message) error { return nil } - if j := h.GetJwt(); j != nil { - // If we're sending installation output but the user does not have the required - // permissions to see the output, don't send it down the line. - if v.Event == server.InstallOutputEvent { - if !j.HasPermission(PermissionReceiveInstall) { - return nil - } + // If we're sending installation output but the user does not have the required + // permissions to see the output, don't send it down the line. + if v.Event == server.InstallOutputEvent { + if !j.HasPermission(PermissionReceiveInstall) { + return nil } + } - // If the user does not have permission to see backup events, do not emit - // them over the socket. - if strings.HasPrefix(string(v.Event), server.BackupCompletedEvent) || - strings.HasPrefix(string(v.Event), server.BackupRestoreCompletedEvent) || - string(v.Event) == server.BackupProgressEvent { - if !j.HasPermission(PermissionReceiveBackups) { - return nil - } + // If the user does not have permission to see backup events, do not emit + // them over the socket. The fork also gates restore-completed and backup + // progress events behind the same permission. + if strings.HasPrefix(string(v.Event), server.BackupCompletedEvent) || + strings.HasPrefix(string(v.Event), server.BackupRestoreCompletedEvent) || + string(v.Event) == server.BackupProgressEvent { + if !j.HasPermission(PermissionReceiveBackups) { + return nil } + } - // If we are sending transfer output, only send it to the user if they have the required permissions. - if v.Event == server.TransferLogsEvent { - if !j.HasPermission(PermissionReceiveTransfer) { - return nil - } + // Activity events carry file paths and remote download URLs, so they need + // the same permissions the Panel requires to read the activity log and the + // server's files. + if v.Event == server.ActivityEvent && !j.HasPermission(PermissionReceiveActivity) { + return nil + } + if v.Event == server.DownloadProgressEvent && !j.HasPermission(PermissionReceiveFiles) { + return nil + } + + // If we are sending transfer output, only send it to the user if they have the required permissions. + if v.Event == server.TransferLogsEvent { + if !j.HasPermission(PermissionReceiveTransfer) { + return nil } } @@ -194,43 +222,67 @@ func (h *Handler) SendJson(v Message) error { // socket user. Do not call this directly unless you are positive a response should be // sent back to the client! func (h *Handler) unsafeSendJson(v interface{}) error { - h.Lock() - defer h.Unlock() + h.writeMu.Lock() + defer h.writeMu.Unlock() + _ = h.Connection.SetWriteDeadline(time.Now().Add(writeTimeout)) return h.Connection.WriteJSON(v) } +// SendThrottled tells the client that messages are being dropped because too +// many were sent. +func (h *Handler) SendThrottled(scope string) error { + return h.unsafeSendJson(Message{Event: ThrottledEvent, Args: []string{scope}}) +} + +// SendClose sends a close message with the given code and reason. +func (h *Handler) SendClose(code int, reason string) error { + h.writeMu.Lock() + defer h.writeMu.Unlock() + + _ = h.Connection.SetWriteDeadline(time.Now().Add(writeTimeout)) + return h.Connection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(code, reason)) +} + // TokenValid checks if the JWT is still valid. func (h *Handler) TokenValid() error { + _, err := h.validJwt() + return err +} + +// validJwt returns the JWT for the websocket if it is still valid. Callers use +// the returned token for every permission check while handling a message, +// rather than loading it again. +func (h *Handler) validJwt() (*tokens.WebsocketPayload, error) { j := h.GetJwt() if j == nil { - return ErrJwtNotPresent + return nil, ErrJwtNotPresent } if err := jwt.ExpirationTimeValidator(time.Now())(&j.Payload); err != nil { - return err + return nil, err } if j.Denylisted() { - return ErrJwtOnDenylist + return nil, ErrJwtOnDenylist } if !j.HasPermission(PermissionConnect) || !j.HasScope(tokens.Websocket) { - return ErrJwtNoConnectPerm + return nil, ErrJwtNoConnectPerm } if h.server.ID() != j.GetServerUuid() { - return ErrJwtUuidMismatch + return nil, ErrJwtUuidMismatch } - return nil + return j, nil } // SendErrorJson sends an error back to the connected websocket instance by checking the permissions // of the token. If the user has the "receive-errors" grant we will send back the actual // error message, otherwise we just send back a standard error message. func (h *Handler) SendErrorJson(msg Message, err error, shouldLog ...bool) error { - j := h.GetJwt() + j, _ := h.validJwt() isJWTError := IsJwtError(err) wsm := Message{ @@ -248,7 +300,7 @@ func (h *Handler) SendErrorJson(msg Message, err error, shouldLog ...bool) error m, u := h.GetErrorMessage(wsm.Args[0]) wsm.Args = []string{m} - if !isJWTError && (len(shouldLog) == 0 || (len(shouldLog) == 1 && shouldLog[0] == true)) { + if !isJWTError && (len(shouldLog) == 0 || (len(shouldLog) == 1 && shouldLog[0])) { h.server.Log().WithFields(log.Fields{"event": msg.Event, "error_identifier": u.String(), "error": err}). Errorf("error processing websocket event \"%s\"", msg.Event) } @@ -266,6 +318,15 @@ func (h *Handler) GetErrorMessage(msg string) (string, uuid.UUID) { return m, u } +// User returns the UUID of the user the websocket is authenticated as, or an +// empty string if it has not authenticated. +func (h *Handler) User() string { + if j := h.GetJwt(); j != nil { + return j.UserUUID + } + return "" +} + // GetJwt returns the JWT for the websocket in a race-safe manner. func (h *Handler) GetJwt() *tokens.WebsocketPayload { h.RLock() @@ -274,6 +335,15 @@ func (h *Handler) GetJwt() *tokens.WebsocketPayload { return h.jwt } +// activity returns the request activity for an action authorized by the given +// token, so that it is attributed to the user the token belongs to even if +// another token has replaced it on the socket since. +func (h *Handler) activity(j *tokens.WebsocketPayload) server.RequestActivity { + h.RLock() + defer h.RUnlock() + return h.ra.SetUser(j.UserUUID) +} + // setJwt sets the JWT for the websocket in a race-safe manner. func (h *Handler) setJwt(token *tokens.WebsocketPayload) { h.Lock() @@ -292,9 +362,13 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { return nil } + // The token that is validated here is the one used for every permission check + // while handling this message. + var j *tokens.WebsocketPayload if m.Event != AuthenticationEvent { - if err := h.TokenValid(); err != nil { - h.unsafeSendJson(Message{ + var err error + if j, err = h.validJwt(); err != nil { + _ = h.unsafeSendJson(Message{ Event: JwtErrorEvent, Args: []string{err.Error()}, }) @@ -310,8 +384,23 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { return err } + // Only tokens issued for this server are stored on the socket. + if token.GetServerUuid() != h.server.ID() { + return ErrJwtUuidMismatch + } + // Check if the user has previously authenticated successfully. newConnection := h.GetJwt() == nil + if newConnection { + h.RLock() + fn := h.onAuthenticated + h.RUnlock() + if fn != nil { + if err := fn(); err != nil { + return err + } + } + } // Previously there was a HasPermission(PermissionConnect) check around this, // however NewTokenPayload will return an error if it doesn't have the connect @@ -370,7 +459,7 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { // Check that they have permission to perform this action if it is needed. if permission, exists := actions[action]; exists { - if !h.GetJwt().HasPermission(permission) { + if !j.HasPermission(permission) { return nil } } @@ -388,7 +477,7 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { } if err == nil { - h.server.SaveActivity(h.ra, models.Event(server.ActivityPowerPrefix+action), nil) + h.server.SaveActivity(h.activity(j), models.Event(server.ActivityPowerPrefix+action), nil) } return err @@ -427,7 +516,7 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { } case SendCommandEvent: { - if !h.GetJwt().HasPermission(PermissionSendCommand) { + if !j.HasPermission(PermissionSendCommand) { return nil } @@ -450,7 +539,7 @@ func (h *Handler) HandleInbound(ctx context.Context, m Message) error { if err := h.server.Environment.SendCommand(strings.Join(m.Args, "")); err != nil { return err } - h.server.SaveActivity(h.ra, server.ActivityConsoleCommand, models.ActivityMeta{ + h.server.SaveActivity(h.activity(j), server.ActivityConsoleCommand, models.ActivityMeta{ "command": strings.Join(m.Args, ""), }) return nil diff --git a/router/websocket/websocket_test.go b/router/websocket/websocket_test.go new file mode 100644 index 000000000..ece735ab3 --- /dev/null +++ b/router/websocket/websocket_test.go @@ -0,0 +1,191 @@ +package websocket + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gbrlsnchs/jwt/v3" + "github.com/gorilla/websocket" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/router/tokens" + "github.com/Rene-Roscher/wings/server" +) + +const ( + serverUuid = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + otherServerUuid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" +) + +var errCommandSent = errors.New("command sent") + +// commandEnvironment is a running environment that records the commands sent +// to it. +type commandEnvironment struct { + environment.ProcessEnvironment + commands atomic.Int64 +} + +func (e *commandEnvironment) State() string { + return environment.ProcessRunningState +} + +func (e *commandEnvironment) SendCommand(string) error { + e.commands.Add(1) + // Returning an error skips saving the activity, which needs a database. + return errCommandSent +} + +func setupHandler(t *testing.T) (*Handler, *commandEnvironment) { + t.Helper() + + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + config.Set(cfg) + + s, err := server.New(nil) + if err != nil { + t.Fatal(err) + } + settings, _ := json.Marshal(map[string]string{"uuid": serverUuid}) + if err := s.SyncWithConfiguration(remote.ServerConfigurationResponse{Settings: settings, ProcessConfiguration: &remote.ProcessConfiguration{}}); err != nil { + t.Fatal(err) + } + env := &commandEnvironment{} + s.Environment = env + + conns := make(chan *websocket.Conn, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + conns <- c + })) + t.Cleanup(srv.Close) + + client, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + go func() { + for { + if _, _, err := client.ReadMessage(); err != nil { + return + } + } + }() + + return &Handler{Connection: <-conns, server: s, limiter: NewLimiter()}, env +} + +func signToken(t *testing.T, forServer string, permissions ...string) (string, *tokens.WebsocketPayload) { + t.Helper() + return signTokenFor(t, forServer, "cccccccc-cccc-4ccc-8ccc-cccccccccccc", permissions...) +} + +func signTokenFor(t *testing.T, forServer string, userUuid string, permissions ...string) (string, *tokens.WebsocketPayload) { + t.Helper() + // Tokens issued before Wings booted are rejected, and the issued time only has + // second precision. + now := time.Now().Add(time.Second * 2) + payload := tokens.WebsocketPayload{ + Payload: jwt.Payload{ + IssuedAt: jwt.NumericDate(now), + ExpirationTime: jwt.NumericDate(now.Add(time.Minute * 10)), + }, + Scoped: tokens.Scoped{Scope: string(tokens.Websocket)}, + UserUUID: userUuid, + ServerUUID: forServer, + Permissions: permissions, + } + b, err := jwt.Sign(&payload, config.GetJwtAlgorithm()) + if err != nil { + t.Fatal(err) + } + parsed, err := NewTokenPayload(b) + if err != nil { + t.Fatal(err) + } + return string(b), parsed +} + +// Activity is attributed to the user whose token authorized it, even if another +// user's token is on the socket since. +func TestActivityIsAttributedToTheAuthorizingToken(t *testing.T) { + h, _ := setupHandler(t) + _, authorizing := signTokenFor(t, serverUuid, "aaaaaaaa-0000-4000-8000-000000000001", PermissionConnect, PermissionSendCommand) + _, other := signTokenFor(t, serverUuid, "bbbbbbbb-0000-4000-8000-000000000002", PermissionConnect) + h.setJwt(other) + + a := h.activity(authorizing).Event(server.ActivityConsoleCommand, nil) + if a.User.String != authorizing.UserUUID { + t.Fatalf("expected activity to be attributed to %s, got %q", authorizing.UserUUID, a.User.String) + } +} + +func TestAuthenticationRejectsTokenForDifferentServer(t *testing.T) { + h, _ := setupHandler(t) + _, current := signToken(t, serverUuid, PermissionConnect) + other, _ := signToken(t, otherServerUuid, PermissionConnect, PermissionSendCommand) + h.setJwt(current) + + err := h.HandleInbound(context.Background(), Message{Event: AuthenticationEvent, Args: []string{other}}) + if !errors.Is(err, ErrJwtUuidMismatch) { + t.Fatalf("expected a token for a different server to be rejected, got %v", err) + } + if h.GetJwt() != current { + t.Fatal("expected the token on the socket to be unchanged") + } +} + +// Messages on a socket are handled concurrently, and commands are only ever +// authorized by a token for the server the socket belongs to. +func TestCommandIsNotAuthorizedByTokenForDifferentServer(t *testing.T) { + if runtime.GOMAXPROCS(0) < 2 { + t.Skip("the messages cannot be handled concurrently with a single CPU") + } + h, env := setupHandler(t) + _, current := signToken(t, serverUuid, PermissionConnect) + other, _ := signToken(t, otherServerUuid, PermissionConnect, PermissionSendCommand) + + deadline := time.Now().Add(time.Second * 5) + for i := 0; i < 20000 && time.Now().Before(deadline); i++ { + h.setJwt(current) + h.limiter = NewLimiter() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = h.HandleInbound(context.Background(), Message{Event: AuthenticationEvent, Args: []string{other}}) + }() + // Vary the order the two messages are handled in. + for n := 0; n < i%64; n++ { + _ = n * n + } + _ = h.HandleInbound(context.Background(), Message{Event: SendCommandEvent, Args: []string{"say hello"}}) + wg.Wait() + + if n := env.commands.Load(); n > 0 { + t.Fatalf("expected the command to be refused (iteration %d, %d commands)", i, n) + } + } +} diff --git a/server/backup.go b/server/backup.go index 6cd1feeae..8ff538de8 100644 --- a/server/backup.go +++ b/server/backup.go @@ -3,8 +3,6 @@ package server import ( "archive/tar" "context" - "crypto/sha256" - "encoding/hex" "fmt" "io" "io/fs" @@ -16,6 +14,7 @@ import ( "emperror.dev/errors" "github.com/apex/log" + cerrdefs "github.com/containerd/errdefs" "github.com/Rene-Roscher/wings/environment" "github.com/Rene-Roscher/wings/internal/progress" @@ -53,23 +52,35 @@ func (s *Server) getServerwideIgnoredFiles() (string, error) { return "", err } defer f.Close() - if st.Mode()&os.ModeSymlink != 0 || st.Size() > 32*1024 { - // Don't read a symlinked ignore file, or a file larger than 32KiB in size. + if st.Mode()&os.ModeSymlink != 0 || st.Size() > filesystem.MaxIgnoreLength { + // Don't read a symlinked ignore file, or a file larger than the maximum list size. return "", nil } b, err := io.ReadAll(f) if err != nil { return "", err } + if err := filesystem.ValidateIgnore(string(b)); err != nil { + return "", errors.WrapIf(err, "backup: invalid .pteroignore file") + } return string(b), nil } +// downloadSizeContextKey is the type of DownloadSizeContextKey; a dedicated type +// avoids collisions with context values set by other packages. +type downloadSizeContextKey struct{} + +// DownloadSizeContextKey is the context key under which the router passes the +// size (int64) of a remote backup download to RestoreBackupWithContext so that +// restore progress can be estimated. +var DownloadSizeContextKey = downloadSizeContextKey{} + // determineActualServerState checks the real container state and returns the appropriate Wings state // This function is thread-safe and handles errors gracefully during state cleanup func (s *Server) determineActualServerState() string { // During cleanup, we should NOT preserve operational states // This function is called to determine the FINAL state after operations complete - + // Check if the container is actually running right now if running, err := s.Environment.IsRunning(s.Context()); err == nil { if running { @@ -127,7 +138,7 @@ func (s *Server) BackupWithContext(ctx context.Context, b backup.BackupInterface EnvironmentState: actualState, BackingUp: &backingUp, }) - + if errors.Is(ctx.Err(), context.Canceled) { s.Log().WithField("new_state", actualState).Info("reset server state after backup cancellation") } else { @@ -288,7 +299,7 @@ func (s *Server) BackupWithContext(ctx context.Context, b backup.BackupInterface "uuid": b.Identifier(), "is_successful": true, "checksum": ad.Checksum, - "checksum_type": "sha1", + "checksum_type": "sha256", "file_size": ad.Size, "panel_notified": false, "notify_error": notifyError.Error(), @@ -328,11 +339,11 @@ func (s *Server) BackupWithRetry(ctx context.Context, b backup.BackupInterface, // Exponential backoff: 30s, 60s, 120s... backoffDuration := time.Duration(30*attempt*attempt) * time.Second s.Log().WithFields(log.Fields{ - "backup_id": b.Identifier(), - "attempt": attempt + 1, + "backup_id": b.Identifier(), + "attempt": attempt + 1, "max_retries": maxRetries + 1, - "backoff": backoffDuration, - "last_error": lastErr, + "backoff": backoffDuration, + "last_error": lastErr, }).Warn("retrying backup after failure") // Wait for backoff period or context cancellation @@ -352,8 +363,8 @@ func (s *Server) BackupWithRetry(ctx context.Context, b backup.BackupInterface, if err == nil { if attempt > 0 { s.Log().WithFields(log.Fields{ - "backup_id": b.Identifier(), - "attempt": attempt + 1, + "backup_id": b.Identifier(), + "attempt": attempt + 1, "total_attempts": attempt + 1, }).Info("backup succeeded after retry") } @@ -361,7 +372,7 @@ func (s *Server) BackupWithRetry(ctx context.Context, b backup.BackupInterface, } lastErr = err - + // Don't retry on context cancellation or unrecoverable errors if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { s.Log().WithFields(log.Fields{ @@ -374,30 +385,30 @@ func (s *Server) BackupWithRetry(ctx context.Context, b backup.BackupInterface, // Log the failed attempt s.Log().WithFields(log.Fields{ - "backup_id": b.Identifier(), - "attempt": attempt + 1, + "backup_id": b.Identifier(), + "attempt": attempt + 1, "max_retries": maxRetries + 1, - "error": err, + "error": err, }).Error("backup attempt failed") } // All attempts failed - emit failure events s.Events().Publish(BackupCompletedEvent, map[string]any{ - "uuid": b.Identifier(), - "is_successful": false, - "error": lastErr.Error(), + "uuid": b.Identifier(), + "is_successful": false, + "error": lastErr.Error(), "total_attempts": maxRetries + 1, }) - + // Also emit as ActivityEvent for persistent logging (WORK.md requirement) s.Events().Publish(ActivityEvent, map[string]any{ - "event": "backup_failed", - "timestamp": time.Now().Unix(), - "backup_id": b.Identifier(), - "server_id": s.ID(), - "error": lastErr.Error(), + "event": "backup_failed", + "timestamp": time.Now().Unix(), + "backup_id": b.Identifier(), + "server_id": s.ID(), + "error": lastErr.Error(), "total_attempts": maxRetries + 1, - "message": fmt.Sprintf("Backup failed after %d attempts: %v", maxRetries+1, lastErr), + "message": fmt.Sprintf("Backup failed after %d attempts: %v", maxRetries+1, lastErr), }) return errors.WithStackIf(lastErr) @@ -413,41 +424,49 @@ func (s *Server) Backup(b backup.BackupInterface) error { return s.BackupWithContext(ctx, b) } -// RestoreBackup calls the Restore function on the provided backup. Once this -// restoration is completed an event is emitted to the websocket to notify the -// Panel that is has been completed. +// RestoreBackupWithContext performs a server backup restore with context for cancellation support. +// This is the primary restore function that should be used for all restore operations. +// +// It calls the Restore function on the provided backup. Once this restoration is +// completed an event is emitted to the websocket to notify the Panel that is has +// been completed. If truncate is set, all of the server's files are deleted once +// it has stopped, before the backup is restored. // // In addition to the websocket event an API call is triggered to notify the // Panel of the new state. -// RestoreBackupWithContext performs a server backup restore with context for cancellation support. -// This is the primary restore function that should be used for all restore operations. -func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupInterface, reader io.ReadCloser) (err error) { - s.Config().SetSuspended(true) - +// +// The server's suspension state is deliberately left untouched (upstream v1.13.4): +// toggling it here would clobber a suspension applied by the Panel while the +// restore is running. Instead the restoring flag is set before waiting for the +// server to stop, which blocks power actions for the whole restore. +func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupInterface, reader io.ReadCloser, truncate bool) (err error) { // CRITICAL: DEFER ORDER (LIFO - Last In First Out) // First defined = Last executed // We want execution order: 1) Final progress, 2) State reset, 3) Panel notify, 4) Resource cleanup // So we define in REVERSE: Resource cleanup, Panel notify, State reset, Final progress - + var progressTracker *SimpleProgressTracker - + // Define FIRST - executes LAST: Resource cleanup defer func() { - s.Config().SetSuspended(false) + // Local backups will not pass a reader through to this function, so check + // first to make sure it is a valid reader before trying to close it. if reader != nil { _ = reader.Close() } s.Log().Debug("restore resource cleanup completed") }() - + // Define SECOND - executes THIRD: Panel notification (needs err value) + // Send an API call to the Panel as soon as this function is done running so that + // the Panel is informed of the restoration status of this backup. defer func() { s.Log().WithField("success", err == nil).Debug("notifying panel of restore status") if rerr := s.client.SendRestorationStatus(s.Context(), b.Identifier(), err == nil); rerr != nil { s.Log().WithField("error", rerr).WithField("backup", b.Identifier()).Error("failed to notify Panel of backup restoration status") } }() - + // Define THIRD - executes SECOND: State reset (MUST happen before final progress) defer func() { // CRITICAL ERROR RECOVERY: Always reset state even if panic occurs @@ -460,33 +479,33 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn s.Environment.SetState(environment.ProcessOfflineState) } }() - + s.Log().WithFields(log.Fields{ - "restoring_before": s.IsRestoring(), + "restoring_before": s.IsRestoring(), "environment_state_before": s.Environment.State(), }).Debug("restore state cleanup starting") - + // Determine correct post-restore state actualState := s.determineActualServerState() restoring := false - + s.Log().WithFields(log.Fields{ - "target_state": actualState, + "target_state": actualState, "target_restoring": restoring, }).Debug("applying atomic state transition for restore cleanup") - + s.ApplyAtomicStateTransition(AtomicStateTransition{ EnvironmentState: actualState, Restoring: &restoring, }) - + s.Log().WithFields(log.Fields{ - "new_state": actualState, - "restoring_after": s.IsRestoring(), + "new_state": actualState, + "restoring_after": s.IsRestoring(), "environment_state_after": s.Environment.State(), }).Info("reset server state after restore") }() - + // Define LAST - executes FIRST: Send final progress (AFTER state is reset!) defer func() { // CRITICAL: Recover from any panic in progress tracking @@ -495,36 +514,46 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn s.Log().WithField("panic", r).Error("panic in final progress tracking - ignored") } }() - + // IMPORTANT: Use the actual error value at defer execution time, not capture time! success := err == nil - + if progressTracker != nil { s.Log().WithFields(log.Fields{ - "success": success, + "success": success, "is_restoring": s.IsRestoring(), - "state": s.Environment.State(), + "state": s.Environment.State(), }).Info("sending final restore progress") progressTracker.SendFinalProgress(success) progressTracker.Close() } - + // Send the restore completed event HERE, while we're still in restore state // This ensures the event is sent before state reset s.Events().Publish(BackupRestoreCompletedEvent, map[string]any{ "successful": success, }) - + // Log the event for debugging s.Log().WithField("successful", success).Info("sent BackupRestoreCompletedEvent") }() // Don't try to restore the server until we have completely stopped the running - // instance, otherwise you'll likely hit all types of write errors due to the - // server being suspended. - if s.Environment.State() != environment.ProcessOfflineState { + // instance, otherwise you'll likely hit all types of write errors. The server + // cannot be started again while it is being restored: the restoring flag is + // set before waiting, so power actions are rejected from here on (this also + // cancels any open SFTP connections). + s.SetRestoring(true) + needsStop := s.Environment.State() != environment.ProcessOfflineState + if s.Environment.State() == environment.ProcessRestoreQueuedState { + // Fork: the queued state masks the real process state, so ask the + // environment directly instead of trying to stop an offline container. + running, rerr := s.Environment.IsRunning(ctx) + needsStop = rerr != nil || running + } + if needsStop { if err = s.Environment.WaitForStop(ctx, 2*time.Minute, false); err != nil { - if !errors.Is(err, os.ErrNotExist) { + if !cerrdefs.IsNotFound(err) && !errors.Is(err, os.ErrNotExist) { return errors.WrapIf(err, "server/backup: restore: failed to wait for container stop") } } @@ -578,19 +607,19 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn // Check if download size was passed through context (for S3 downloads) var downloadSize int64 - if ctxSize := ctx.Value("download_size"); ctxSize != nil { + if ctxSize := ctx.Value(DownloadSizeContextKey); ctxSize != nil { if size, ok := ctxSize.(int64); ok && size > 0 { downloadSize = size s.Log().WithField("download_size", downloadSize).Debug("using download size from context for restore progress") } } - + // Try to get backup file size for percentage calculation backupSize, err := b.Details(s.Context(), nil) if err != nil { s.Log().WithField("error", err).Debug("failed to get backup details for size") } - + // Determine the best size to use for progress tracking var estimatedTotal int64 if downloadSize > 0 { @@ -608,7 +637,7 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn estimatedTotal = int64(10 * 1024 * 1024 * 1024) // 10GB estimate s.Log().WithField("estimated_restore_size", estimatedTotal).Info("using estimated size for restore progress (backup size unavailable)") } - + restoreProgress.SetTotal(uint64(estimatedTotal)) progressTracker = NewSimpleProgressTracker(ctx, s, b.Identifier(), "restore", restoreProgress) @@ -629,6 +658,16 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn } } + // Delete the server's files only now: the server has stopped, the backup has + // been located and (for remote restores) its format has been detected, so a + // restore that cannot happen leaves the files as they are (upstream v1.13.4). + if truncate { + s.Log().Info("deleting server files before restoring backup") + if err = s.Filesystem().TruncateRootDirectory(); err != nil { + return errors.WrapIf(err, "server/backup: restore: failed to delete server files") + } + } + // Attempt to restore the backup to the server by running through each entry // in the file one at a time and writing them to the disk. s.Log().Debug("starting file writing process for backup restoration") @@ -640,9 +679,7 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn // Local backup: let backup interface handle its own file reading restoreReader = nil } - - // MEMORY SAFETY: Set maximum buffer size for file operations (10MB) - const maxBufferSize = 10 * 1024 * 1024 + buffer := make([]byte, 32*1024) // 32KB buffer for streaming // Track restore statistics for validation @@ -655,7 +692,7 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn err = b.Restore(ctx, restoreReader, func(file string, info fs.FileInfo, r io.ReadCloser) error { defer r.Close() //s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) - + // Use buffer to mark as used (for memory-safe streaming in future) _ = buffer @@ -672,38 +709,15 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn restoreStats.totalSize += info.Size() } - // Handle directories and files differently - if info.IsDir() { - // For directories, create the directory structure using the underlying UnixFS - created, err := s.Filesystem().UnixFS().MkdirAll(file, ufs.FileMode(info.Mode())) - if err != nil { - return err - } - // Chown every directory we just created so restored directories are - // owned by the server user instead of the user Wings runs as. - for _, dir := range created { - if err := s.Filesystem().Chown(dir); err != nil { - return err - } - } - // Set directory timestamps - atime := info.ModTime() - return s.Filesystem().Chtimes(file, atime, atime) - } - - // For regular files, write the content - // TODO: since this will be called a lot, it may be worth adding an optimized - // Write with Chtimes method to the UnixFS that is able to re-use the - // same dirfd and file name. - if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil { + if err := s.restoreEntry(file, info, r); err != nil { return err } - atime := info.ModTime() // Send ultra-live progress update AFTER successful write - updateProgress(info.Size()) - - return s.Filesystem().Chtimes(file, atime, atime) + if info.Mode().IsRegular() { + updateProgress(info.Size()) + } + return nil }) // Basic restore validation @@ -727,8 +741,8 @@ func (s *Server) RestoreBackupWithContext(ctx context.Context, b backup.BackupIn // RestoreBackup performs a server backup restore with the server's context. // This method is kept for backward compatibility. New code should use RestoreBackupWithContext. -func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) error { - return s.RestoreBackupWithContext(s.Context(), b, reader) +func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser, truncate bool) error { + return s.RestoreBackupWithContext(s.Context(), b, reader, truncate) } // generateBackupWithProgress creates a backup with progress tracking and context support @@ -744,244 +758,7 @@ func (s *Server) generateBackupWithProgress(ctx context.Context, b backup.Backup } // Fallback to original Generate method if backup type is unknown - ad, err := b.Generate(ctx, s.Filesystem(), ignored) - if err != nil { - return nil, err - } - - // Quick integrity validation for any backup type - CRITICAL: Fail backup on validation errors - if err := s.validateBackupIntegrity(b); err != nil { - s.Log().WithError(err).Error("backup integrity validation failed - backup is corrupted") - return ad, errors.Wrap(err, "backup integrity validation failed") - } - - // Content integrity validation (file/directory count check) - CRITICAL: Fail backup on validation errors - backupPath := b.Path() // Works for all backup types - if err := s.validateBackupContent(backupPath, s.Filesystem().Path()); err != nil { - s.Log().WithError(err).Error("backup content validation failed - backup is incomplete") - return ad, errors.Wrap(err, "backup content validation failed") - } else { - s.Log().Debug("backup content validation passed - backup is complete") - } - - return ad, nil -} - -// validateBackupIntegrity performs minimal integrity checks on backup file -func (s *Server) validateBackupIntegrity(b backup.BackupInterface) error { - backupPath := "" - - // Get backup path based on type - if localBackup, ok := b.(*backup.LocalBackup); ok { - backupPath = localBackup.Path() - } else { - // For S3 backups, we can't validate local file - return nil - } - - // Basic file existence and size check - stat, err := os.Stat(backupPath) - if err != nil { - return errors.Wrap(err, "backup file not accessible") - } - - // Archive must be at least 20 bytes (minimum GZIP + TAR headers) - if stat.Size() < 20 { - return errors.New("backup file suspiciously small - may be corrupt") - } - - // Quick magic bytes check for GZIP - f, err := os.Open(backupPath) - if err != nil { - return err - } - defer f.Close() - - magic := make([]byte, 2) - if n, err := f.Read(magic); err != nil || n < 2 { - return errors.New("cannot read backup file header") - } - - // Check for GZIP magic bytes (0x1f, 0x8b) - if magic[0] == 0x1f && magic[1] == 0x8b { - // Valid GZIP - return nil - } - - // Check for uncompressed TAR (less common but possible) - if _, err := f.Seek(0, 0); err != nil { - return err - } - - // Read TAR header area - tarTest := make([]byte, 512) - if n, err := f.Read(tarTest); err != nil || n < 512 { - return errors.New("backup file too short for valid TAR") - } - - // Very basic TAR validation - check for reasonable header structure - // TAR headers have specific patterns at specific offsets - if tarTest[156] == '0' || tarTest[156] == '5' { // Regular file or directory - return nil - } - - return errors.New("backup file format not recognized - may be corrupt") -} - -// validateBackupContent performs fast file/directory count validation between original and backup -func (s *Server) validateBackupContent(backupPath, serverPath string) error { - // 1. Count original files and directories (fast directory walk) - originalStats, err := s.countServerFilesAndDirs(serverPath) - if err != nil { - return errors.Wrap(err, "failed to count original server files") - } - - // 2. Count backup entries (TAR header scan only, no extraction) - backupStats, err := s.countBackupEntries(backupPath) - if err != nil { - return errors.Wrap(err, "failed to count backup entries") - } - - // Generate SHA1 checksums for debug logging - backupChecksum := "unknown" - serverChecksum := "unknown" - - // Get backup file SHA1 (reuse existing checksum method) - if backupFile, err := os.Open(backupPath); err == nil { - hasher := sha256.New() - if _, err := io.Copy(hasher, backupFile); err == nil { - backupChecksum = hex.EncodeToString(hasher.Sum(nil)) - } - backupFile.Close() - } - - // Get server directory content SHA1 (walk files and hash content) - if serverHash := sha256.New(); serverHash != nil { - err := filepath.Walk(serverPath, func(path string, info os.FileInfo, err error) error { - if err != nil || path == serverPath || info.IsDir() { - return nil // Skip errors, root, and directories - } - - relPath, _ := filepath.Rel(serverPath, path) - serverHash.Write([]byte(relPath)) // Include path in hash - - if file, err := os.Open(path); err == nil { - io.Copy(serverHash, file) - file.Close() - } - return nil - }) - - if err == nil { - serverChecksum = hex.EncodeToString(serverHash.Sum(nil)) - } - } - - s.Log().WithFields(log.Fields{ - "original_files": originalStats.FileCount, - "original_dirs": originalStats.DirCount, - "backup_files": backupStats.FileCount, - "backup_dirs": backupStats.DirCount, - "backup_sha256": backupChecksum, - "server_content_sha256": serverChecksum, - }).Debug("backup content validation stats") - - // 3. Compare file counts - if originalStats.FileCount != backupStats.FileCount { - return errors.Errorf("backup file count mismatch: expected %d files, backup contains %d files", - originalStats.FileCount, backupStats.FileCount) - } - - // 4. Compare directory counts - if originalStats.DirCount != backupStats.DirCount { - return errors.Errorf("backup directory count mismatch: expected %d directories, backup contains %d directories", - originalStats.DirCount, backupStats.DirCount) - } - - return nil -} - -// fileStats holds counts for validation -type fileStats struct { - FileCount int - DirCount int -} - -// countServerFilesAndDirs counts files and directories in server filesystem -func (s *Server) countServerFilesAndDirs(serverPath string) (*fileStats, error) { - stats := &fileStats{} - - err := filepath.Walk(serverPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - // Skip unreadable files/dirs but continue counting - return nil - } - - // Skip the root directory itself - if path == serverPath { - return nil - } - - if info.IsDir() { - stats.DirCount++ - } else { - stats.FileCount++ - } - - return nil - }) - - return stats, err -} - -// countBackupEntries counts entries in TAR archive by scanning headers only (no extraction) -func (s *Server) countBackupEntries(backupPath string) (*fileStats, error) { - stats := &fileStats{} - - f, err := os.Open(backupPath) - if err != nil { - return nil, err - } - defer f.Close() - - // Auto-detect compression and create appropriate reader - format, detectedReader, err := filesystem.DetectCompressionFormat(io.NopCloser(f)) - if err != nil { - return nil, errors.Wrap(err, "failed to detect backup compression format") - } - - decompressedReader, err := filesystem.CreateDecompressor(detectedReader, format) - if err != nil { - return nil, errors.Wrap(err, "failed to create decompressor") - } - defer decompressedReader.Close() - - // Scan TAR headers (no content reading) - tarReader := tar.NewReader(decompressedReader) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, errors.Wrap(err, "failed to read TAR header") - } - - // Skip problematic root entries (same logic as restore) - if header.Name == "." || header.Name == "" || header.Name == "/" || - header.Name == "./" || strings.HasPrefix(header.Name, "../") { - continue - } - - // Count based on header type - if header.FileInfo().IsDir() { - stats.DirCount++ - } else { - stats.FileCount++ - } - } - - return stats, nil + return b.Generate(ctx, s.Filesystem(), ignored) } // generateLocalBackupWithProgress creates a local backup with progress tracking and context support @@ -1021,7 +798,7 @@ func (s *Server) generateLocalBackupWithProgress(ctx context.Context, b *backup. func (s *Server) generateS3BackupWithProgress(ctx context.Context, b *backup.S3Backup, ignored string, progressInstance *progress.Progress, progressTracker *SimpleProgressTracker) (*backup.ArchiveDetails, error) { // S3 PROGRESS: 80/20 split pattern for S3 backups // Archive creation = 80%, Upload = 20% - + // Configure tracker for S3 80/20 mode from the start if progressTracker != nil { // Mark as S3 immediately, archive size will be set later @@ -1054,7 +831,7 @@ func (s *Server) generateS3BackupWithProgress(ctx context.Context, b *backup.S3B // Phase 2: S3 upload with REAL progress tracking (remaining 20%) s.Log().Debug("S3 upload phase starting with real progress tracking") - + // Set up real progress tracking for S3 upload with WebSocket callback if progressInstance != nil && progressTracker != nil { b.WithUploadProgress(progressInstance) @@ -1073,3 +850,55 @@ func (s *Server) generateS3BackupWithProgress(ctx context.Context, b *backup.S3B s.Log().WithField("backup", b.Identifier()).Info("S3 backup upload completed") return ad, nil } + +// restoreEntry restores a single entry of a backup archive. Entries are restored +// as the type they have in the archive (upstream v1.13.4): directories, symlinks +// and regular files; hard links, devices and other special files are skipped. +// +// Fork divergence: directories keep their archived mode and modification time, +// and every directory created along the way is chowned to the server user. +func (s *Server) restoreEntry(file string, info fs.FileInfo, r io.Reader) error { + switch { + case info.IsDir(): + // For directories, create the directory structure using the underlying UnixFS + created, err := s.Filesystem().UnixFS().MkdirAll(file, ufs.FileMode(info.Mode())) + if err != nil { + return err + } + // Chown every directory we just created so restored directories are + // owned by the server user instead of the user Wings runs as. + for _, dir := range created { + if err := s.Filesystem().Chown(dir); err != nil { + return err + } + } + // Set directory timestamps + atime := info.ModTime() + return s.Filesystem().Chtimes(file, atime, atime) + case info.Mode()&fs.ModeSymlink != 0: + h, ok := info.Sys().(*tar.Header) + if !ok || h.Linkname == "" { + return nil + } + if err := s.Filesystem().CreateDirectory(filepath.Dir(file), "/"); err != nil { + return err + } + if err := s.Filesystem().Symlink(h.Linkname, file); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + return nil + case !info.Mode().IsRegular(): + // Hard links, devices and other special files are not restored. + return nil + } + + // For regular files, write the content + // TODO: since this will be called a lot, it may be worth adding an optimized + // Write with Chtimes method to the UnixFS that is able to re-use the + // same dirfd and file name. + if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil { + return err + } + atime := info.ModTime() + return s.Filesystem().Chtimes(file, atime, atime) +} diff --git a/server/backup/backup.go b/server/backup/backup.go index 8eaf7bc7a..0329e7ed1 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -13,7 +13,6 @@ import ( "emperror.dev/errors" "github.com/apex/log" "github.com/google/uuid" - "github.com/mholt/archives" "golang.org/x/sync/errgroup" "github.com/Rene-Roscher/wings/config" @@ -21,12 +20,6 @@ import ( "github.com/Rene-Roscher/wings/server/filesystem" ) -var format = archives.CompressedArchive{ - Compression: archives.Gz{}, - Archival: archives.Tar{}, - Extraction: archives.Tar{}, -} - type AdapterType string const ( @@ -53,7 +46,7 @@ type BackupInterface interface { Generate(context.Context, *filesystem.Filesystem, string) (*ArchiveDetails, error) // Ignored returns the ignored files for this backup instance. Ignored() string - // Checksum returns a SHA1 checksum for the generated backup. + // Checksum returns a SHA256 checksum for the generated backup. Checksum() ([]byte, error) // Size returns the size of the generated backup. Size() (int64, error) @@ -83,6 +76,11 @@ type Backup struct { client remote.Client adapter AdapterType logContext map[string]interface{} + + // foundPath is set by LocateLocal when an existing local backup was found + // under a legacy extension (.tar). Path() returns it, so Size, Checksum and + // Details read the same file that Restore and Remove use. + foundPath string } func (b *Backup) SetClient(c remote.Client) { @@ -106,12 +104,19 @@ func (b *Backup) validateIdentifier() error { if err != nil { return err } - b.Uuid = identifier + // Only write when the identifier actually changes, so that concurrent readers + // (e.g. Checksum and Size running in parallel from Details) do not race. + if b.Uuid != identifier { + b.Uuid = identifier + } return nil } // Path returns the path for this specific backup. func (b *Backup) Path() string { + if b.foundPath != "" { + return b.foundPath + } identifier, err := b.normalizedIdentifier() if err != nil { identifier = path.Base(b.Identifier()) @@ -119,14 +124,6 @@ func (b *Backup) Path() string { return path.Join(config.Get().System.BackupDirectory, identifier+".tar.gz") } -// PathForLocalBackup returns the path for a LocalBackup, checking for foundPath override -func (b *Backup) PathForLocalBackup(foundPath string) string { - if foundPath != "" { - return foundPath // Use discovered path for backward compatibility - } - return b.Path() // Use standard path generation -} - // Size returns the size of the generated backup. func (b *Backup) Size() (int64, error) { if err := b.validateIdentifier(); err != nil { @@ -164,8 +161,13 @@ func (b *Backup) Checksum() ([]byte, error) { // Details returns both the checksum and size of the archive currently stored on // the disk to the caller. func (b *Backup) Details(ctx context.Context, parts []remote.BackupPart) (*ArchiveDetails, error) { + // Normalize the identifier once before the checksum and size are computed + // concurrently below. + if err := b.validateIdentifier(); err != nil { + return nil, err + } ad := ArchiveDetails{ChecksumType: "sha256", Parts: parts} - g, ctx := errgroup.WithContext(ctx) + var g errgroup.Group g.Go(func() error { resp, err := b.Checksum() diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index 975a0ccde..4884f3122 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -5,11 +5,8 @@ import ( "io" "os" "path" - "path/filepath" - "strings" "emperror.dev/errors" - "github.com/apex/log" "github.com/juju/ratelimit" "github.com/mholt/archives" @@ -20,8 +17,6 @@ import ( type LocalBackup struct { Backup - // foundPath overrides Path() for backward compatibility when locating existing backups - foundPath string } var _ BackupInterface = (*LocalBackup)(nil) @@ -34,7 +29,6 @@ func NewLocal(client remote.Client, uuid string, ignore string) *LocalBackup { Ignore: ignore, adapter: LocalBackupAdapter, }, - foundPath: "", // Initialize foundPath } } @@ -55,38 +49,20 @@ func LocateLocal(client remote.Client, uuid string) (*LocalBackup, os.FileInfo, } return b, st, nil } - - // BACKWARD COMPATIBILITY: Try other formats if current format not found + + // BACKWARD COMPATIBILITY: uncompressed backups are stored as .tar. if os.IsNotExist(err) { - // Try all possible extensions for backward compatibility - possibleExtensions := []string{".tar.gz", ".tar"} - baseDir := config.Get().System.BackupDirectory - - for _, ext := range possibleExtensions { - backupPath := path.Join(baseDir, uuid+ext) - if st, err := os.Stat(backupPath); err == nil { - if st.IsDir() { - return nil, nil, errors.New("invalid archive, is directory") - } - - // Create backup instance with found path - backup := NewLocal(client, uuid, "") - // Override the path to the actually found file - backup.foundPath = backupPath - return backup, st, nil + tarPath := path.Join(config.Get().System.BackupDirectory, b.Uuid+".tar") + if st, terr := os.Stat(tarPath); terr == nil { + if st.IsDir() { + return nil, nil, errors.New("invalid archive, is directory") } + b.foundPath = tarPath + return b, st, nil } } - - return nil, nil, err -} -// Path returns the path for this LocalBackup, considering foundPath override -func (b *LocalBackup) Path() string { - if b.foundPath != "" { - return b.foundPath // Use discovered path for backward compatibility - } - return b.Backup.Path() // Use standard path generation + return nil, nil, err } // Remove removes a backup from the system. @@ -144,22 +120,22 @@ func (b *LocalBackup) Restore(ctx context.Context, _ io.Reader, callback Restore if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 { reader = ratelimit.Reader(f, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit)) } - + // Wrap reader in NopCloser to satisfy ReadCloser interface readCloser := io.NopCloser(reader) - + // Auto-detect compression format and decompress format, detectedReader, err := filesystem.DetectCompressionFormat(readCloser) if err != nil { return errors.WrapIf(err, "failed to detect backup compression format") } - + decompressedReader, err := filesystem.CreateDecompressor(detectedReader, format) if err != nil { return errors.WrapIf(err, "failed to create decompressor for backup") } defer decompressedReader.Close() - + // Use the mholt/archives package to extract TAR archive tarFormat := archives.Tar{} if err := tarFormat.Extract(ctx, decompressedReader, func(ctx context.Context, f archives.FileInfo) error { @@ -175,113 +151,3 @@ func (b *LocalBackup) Restore(ctx context.Context, _ io.Reader, callback Restore } return nil } - -// CleanupBackupFilesForServer removes all local backup files associated with a server -// This function is called during server deletion to prevent orphaned backup files -func CleanupBackupFilesForServer(serverID string) error { - backupDir := config.Get().System.BackupDirectory - logger := log.WithFields(log.Fields{ - "server_id": serverID, - "backup_dir": backupDir, - }) - - // List all files in backup directory - files, err := os.ReadDir(backupDir) - if err != nil { - if os.IsNotExist(err) { - logger.Debug("backup directory does not exist, nothing to clean up") - return nil - } - return errors.WrapIf(err, "failed to read backup directory") - } - - var removedFiles []string - var failedRemovals []string - - // Iterate through all files and find ones belonging to this server - for _, file := range files { - if file.IsDir() { - continue - } - - fileName := file.Name() - - // Check if this file belongs to the server by examining filename patterns - // Backup files are typically named: {backup-uuid}.{extension} - // We need to check if this is actually a backup file for our server - // This is a conservative approach - we only remove files that are clearly backup files - - // Skip files that don't look like backup files - if !isBackupFile(fileName) { - continue - } - - // Extract backup UUID from filename (before first dot) - parts := strings.Split(fileName, ".") - if len(parts) < 2 { - continue // Not a valid backup file format - } - - backupUUID := parts[0] - - // Skip if the UUID doesn't look valid (should be 36 characters for UUID) - if len(backupUUID) != 36 { - continue - } - - filePath := filepath.Join(backupDir, fileName) - - // For safety, we should ideally check if this backup belongs to the server - // However, we don't have a reliable way to determine ownership without - // querying the panel or parsing backup metadata - // - // For now, we use a conservative approach: only remove files if they match - // our known backup file patterns and are in the correct directory - - logger.WithField("file", fileName).Debug("found potential backup file, attempting removal") - - if err := os.Remove(filePath); err != nil { - logger.WithError(err).WithField("file", fileName).Error("failed to remove backup file") - failedRemovals = append(failedRemovals, fileName) - } else { - logger.WithField("file", fileName).Info("removed backup file") - removedFiles = append(removedFiles, fileName) - } - } - - // Log summary - if len(removedFiles) > 0 { - logger.WithFields(log.Fields{ - "removed_count": len(removedFiles), - "removed_files": removedFiles, - }).Info("cleaned up backup files for server") - } - - if len(failedRemovals) > 0 { - logger.WithFields(log.Fields{ - "failed_count": len(failedRemovals), - "failed_files": failedRemovals, - }).Warn("some backup files could not be removed") - return errors.New("failed to remove some backup files") - } - - return nil -} - -// isBackupFile checks if a filename looks like a backup file -func isBackupFile(filename string) bool { - // Common backup file extensions - backupExtensions := []string{ - ".tar.gz", ".tar", ".gz", - } - - lowerName := strings.ToLower(filename) - - for _, ext := range backupExtensions { - if strings.HasSuffix(lowerName, ext) { - return true - } - } - - return false -} diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 43e3f7a9a..cbbcf5b8c 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -3,6 +3,7 @@ package backup import ( "bytes" "context" + "crypto/tls" "fmt" "io" "net/http" @@ -89,7 +90,10 @@ func (s *S3Backup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ig defer func() { if success { - s.Remove() // Only remove on successful upload + // Only remove on successful upload + if err := s.Remove(); err != nil && !errors.Is(err, os.ErrNotExist) { + s.log().WithField("error", err).Warn("failed to remove local copy of backup") + } } else { // Clean up orphaned S3 parts on failure if len(uploadedParts) > 0 { @@ -161,7 +165,7 @@ func (s *S3Backup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ig // If the callback returns an error the entire process is stopped. func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCallback) error { s.log().Debug("S3 restore: starting restore process") - + // CRITICAL: The reader provided here is ALREADY DECOMPRESSED by the server layer! // The server's RestoreBackupWithContext method handles: // 1. Format detection (gzip, etc.) @@ -169,33 +173,33 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal // 3. Passing us the clean TAR stream // // We should NOT attempt format detection or decompression here! - + // Start with the provided reader (already decompressed TAR stream) finalReader := r - + // Apply write rate limiting to prevent disk overload if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 { s.log().WithField("write_limit_mb", writeLimit/1024/1024).Debug("S3 restore: applying write rate limit") finalReader = ratelimit.Reader(r, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit)) } - + // Note: Download progress tracking doesn't make sense here because: // 1. We're receiving an already-decompressed stream // 2. The actual download happens in the router layer // 3. The size would be the decompressed size, not download size // Progress tracking for restore happens at the file extraction level in the server layer - + s.log().Debug("S3 restore: starting TAR archive extraction") // Use the mholt/archives package to extract TAR archive // The reader is already decompressed, just extract the TAR tarFormat := archives.Tar{} fileCount := 0 totalBytes := uint64(0) - + if err := tarFormat.Extract(ctx, finalReader, func(ctx context.Context, f archives.FileInfo) error { fileCount++ totalBytes += uint64(f.Size()) - + // Log every 100 files or every 100MB to track progress if fileCount%100 == 0 || totalBytes%(100*1024*1024) < uint64(f.Size()) { s.log().WithFields(log.Fields{ @@ -204,11 +208,11 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal "current_file": f.NameInArchive, }).Debug("S3 restore: extraction progress") } - + r, err := f.Open() if err != nil { s.log().WithFields(log.Fields{ - "file": f.NameInArchive, + "file": f.NameInArchive, "error": err, }).Error("S3 restore: failed to open file from archive") return err @@ -219,12 +223,12 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal // The callback should be fast and context-aware itself if err := callback(f.NameInArchive, f.FileInfo, r); err != nil { s.log().WithFields(log.Fields{ - "file": f.NameInArchive, + "file": f.NameInArchive, "error": err, }).Error("S3 restore: callback failed for file") return err } - + // Check context after each file select { case <-ctx.Done(): @@ -233,17 +237,17 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal default: // Continue processing } - + return nil }); err != nil { s.log().WithFields(log.Fields{ "files_processed": fileCount, "total_bytes_mb": totalBytes / (1024 * 1024), - "error": err, + "error": err, }).Error("S3 restore: TAR extraction failed") return err } - + s.log().WithFields(log.Fields{ "files_processed": fileCount, "total_bytes_mb": totalBytes / (1024 * 1024), @@ -286,7 +290,7 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) return uploader.uploadedParts, ctx.Err() default: } - + // Get the size for the current part. var partSize int64 if i+1 < len(urls.Parts) { @@ -316,9 +320,9 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) type s3FileUploader struct { io.ReadCloser - client *http.Client - uploadedParts []remote.BackupPart - progressTracker ProgressTracker + client *http.Client + uploadedParts []remote.BackupPart + progressTracker ProgressTracker progressCallback func() } @@ -333,8 +337,12 @@ func newS3FileUploader(file io.ReadCloser) *s3FileUploader { client: &http.Client{ Timeout: time.Hour * 2, Transport: &http.Transport{ - // Force HTTP/1.1 to ensure streaming uploads work properly + Proxy: http.ProxyFromEnvironment, + // Force HTTP/1.1 to ensure streaming uploads work properly. A non-nil, + // empty TLSNextProto is what actually disables HTTP/2; ForceAttemptHTTP2 + // alone does not. ForceAttemptHTTP2: false, + TLSNextProto: map[string]func(string, *tls.Conn) http.RoundTripper{}, // Disable buffering for request bodies DisableCompression: true, // Increase idle connections for better performance @@ -381,12 +389,12 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 if size <= 0 || size > (5*1024*1024*1024) { // Max 5GB per part (S3 limit) return "", errors.New("backup: invalid part size for S3 upload") } - + // For parts <=100MB, buffer for retry support. For larger parts, accept retry failure. const maxBufferSize = 100 * 1024 * 1024 // 100MB threshold var partReader io.Reader var canRetry bool - + if size <= maxBufferSize { // Small part - buffer it for retry support partData := make([]byte, size) @@ -399,7 +407,7 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 } partReader = bytes.NewReader(partData) canRetry = true - + // Don't update progress here for buffered parts! // Progress will be simulated during actual upload to show network transfer // This prevents the progress bar from jumping to completion before upload starts @@ -408,17 +416,17 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 partReader = io.LimitReader(fu.ReadCloser, size) canRetry = false } - + var etag string attempt := 0 err := backoff.Retry(func() error { attempt++ - + // For large parts, only allow one attempt if !canRetry && attempt > 1 { return backoff.Permanent(errors.New("backup: cannot retry large part upload")) } - + // Create new request for each attempt r, err := http.NewRequestWithContext(ctx, http.MethodPut, part, nil) if err != nil { @@ -436,7 +444,9 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 if canRetry { // Reset reader for retry if br, ok := partReader.(*bytes.Reader); ok { - br.Seek(0, 0) + if _, err := br.Seek(0, io.SeekStart); err != nil { + return errors.Wrap(err, "backup: could not rewind part for retry") + } } // IMPORTANT: Add progress tracking for buffered uploads too! // This simulates progress during network transfer @@ -458,7 +468,7 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 "part_size_mb": size / (1024 * 1024), "has_callback": fu.progressCallback != nil, }).Debug("S3: Setting up progress tracking for large part upload") - + progressReader := NewProgressReader(partReader, fu.progressTracker) if fu.progressCallback != nil { progressReader.WithCallback(fu.progressCallback) @@ -468,7 +478,7 @@ func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int6 r.Body = Reader{Reader: partReader} } } - + res, err := fu.client.Do(r) if err != nil { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { @@ -555,46 +565,46 @@ func (pr *ProgressReader) Read(p []byte) (n int, err error) { if len(p) > maxChunk { p = p[:maxChunk] } - + n, err = pr.reader.Read(p) - + // Debug: Log every read call if n > 0 && pr.progress != nil { totalRead := pr.progress.Written() log.WithFields(log.Fields{ - "bytes_read": n, - "total_read": totalRead, + "bytes_read": n, + "total_read": totalRead, "buffer_size": len(p), - "is_eof": err == io.EOF, + "is_eof": err == io.EOF, }).Debug("S3 ProgressReader: Read called") } - + if n > 0 && pr.progress != nil { pr.mutex.Lock() defer pr.mutex.Unlock() // CRITICAL: Always unlock, even on panic - + pr.progress.AddWritten(uint64(n)) - + // Trigger callback if set, with intelligent throttling if pr.callback != nil { now := time.Now().UnixNano() - + // Dynamic throttling based on data rate: // - For fast uploads (>10MB/s): throttle to 100ms - // - For normal uploads: throttle to 250ms + // - For normal uploads: throttle to 250ms // - Always send first and last update bytesWritten := pr.progress.Written() isFirst := pr.lastCallback == 0 isLast := err == io.EOF - + // Debug log first read if isFirst { log.WithFields(log.Fields{ - "bytes_read": n, + "bytes_read": n, "total_written": bytesWritten, }).Debug("S3 ProgressReader: First read from upload stream") } - + var throttleInterval int64 = 250_000_000 // Default 250ms if !isFirst && pr.lastCallback > 0 { // Calculate upload speed based on bytes since last callback @@ -608,27 +618,28 @@ func (pr *ProgressReader) Read(p []byte) (n int, err error) { } } } - + shouldSend := isFirst || isLast || (now-pr.lastCallback >= throttleInterval) - + if shouldSend { pr.lastCallback = now pr.lastBytesWritten = bytesWritten - + // Debug log callback trigger log.WithFields(log.Fields{ - "is_first": isFirst, - "is_last": isLast, + "is_first": isFirst, + "is_last": isLast, "bytes_written": bytesWritten, - "should_send": shouldSend, - "throttle_ms": throttleInterval / 1_000_000, + "should_send": shouldSend, + "throttle_ms": throttleInterval / 1_000_000, }).Debug("S3 ProgressReader: Callback check") - + // Recover from panic in callback - progress events must never break uploads func() { defer func() { + // Ignore panics - progress is not critical. if r := recover(); r != nil { - // Silently ignore panics - progress is not critical + log.WithField("panic", r).Debug("S3 ProgressReader: progress callback panicked") } }() pr.callback() diff --git a/server/backup/compression.go b/server/backup/compression.go index a7f402aa4..8eda3c212 100644 --- a/server/backup/compression.go +++ b/server/backup/compression.go @@ -18,16 +18,16 @@ const ( type CompressionAdapter interface { // Format returns the compression format identifier Format() CompressionFormat - + // Extension returns the file extension for this format Extension() string - + // ContentTypes returns the list of valid MIME types for this format ContentTypes() []string - + // IsSupported checks if the format is supported for backup operations IsSupported() bool - + // Description returns a human-readable description of the format Description() string } @@ -88,12 +88,12 @@ func NewCompressionRegistry() *CompressionRegistry { registry := &CompressionRegistry{ adapters: make(map[CompressionFormat]CompressionAdapter), } - + // Register default compression formats registry.Register(&gzipAdapter{}) registry.Register(&tarAdapter{}) registry.Register(&noneAdapter{}) - + return registry } @@ -113,7 +113,7 @@ func (r *CompressionRegistry) GetByContentType(contentType string) (CompressionA // Normalize content type ctBase := strings.Split(contentType, ";")[0] ctBase = strings.TrimSpace(strings.ToLower(ctBase)) - + for _, adapter := range r.adapters { for _, ct := range adapter.ContentTypes() { if ct == ctBase { @@ -121,20 +121,20 @@ func (r *CompressionRegistry) GetByContentType(contentType string) (CompressionA } } } - + return nil, false } // GetByExtension returns the adapter that matches the given file extension func (r *CompressionRegistry) GetByExtension(extension string) (CompressionAdapter, bool) { extension = strings.ToLower(extension) - + for _, adapter := range r.adapters { if adapter.Extension() == extension { return adapter, true } } - + return nil, false } @@ -162,4 +162,4 @@ var DefaultCompressionRegistry = NewCompressionRegistry() // This function uses the CompressionRegistry for extensible format support func IsValidBackupContentType(contentType string) bool { return DefaultCompressionRegistry.IsValidContentType(contentType) -} \ No newline at end of file +} diff --git a/server/backup/content_validation_test.go b/server/backup/content_validation_test.go index f03cd4e28..ef0d72390 100644 --- a/server/backup/content_validation_test.go +++ b/server/backup/content_validation_test.go @@ -18,24 +18,24 @@ func TestContentValidationFileCount(t *testing.T) { tempDir := t.TempDir() originalDir := filepath.Join(tempDir, "original") backupFile := filepath.Join(tempDir, "backup.tar.gz") - + require.NoError(t, os.MkdirAll(originalDir, 0755)) - + // Create test structure testStructure := map[string]string{ - "file1.txt": "content1", - "file2.txt": "content2", - "level1/file3.txt": "content3", - "level1/level2/file4.txt": "content4", - "empty_dir/.keep": "", - "level1/empty_subdir/.keep": "", + "file1.txt": "content1", + "file2.txt": "content2", + "level1/file3.txt": "content3", + "level1/level2/file4.txt": "content4", + "empty_dir/.keep": "", + "level1/empty_subdir/.keep": "", } - + // Create original files and directories for path, content := range testStructure { fullPath := filepath.Join(originalDir, path) require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) - + if strings.HasSuffix(path, ".keep") { // Create and remove .keep to leave empty directory require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) @@ -44,38 +44,38 @@ func TestContentValidationFileCount(t *testing.T) { require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) } } - + // Test 1: Create correct backup - should validate successfully t.Run("Correct Backup Validates", func(t *testing.T) { err := createCorrectBackup(originalDir, backupFile) require.NoError(t, err) - + // Test validation functions directly originalStats, err := countServerFiles(originalDir) require.NoError(t, err) - + backupStats, err := countBackupEntries(backupFile) require.NoError(t, err) - + assert.Equal(t, originalStats.FileCount, backupStats.FileCount, "File counts should match") assert.Equal(t, originalStats.DirCount, backupStats.DirCount, "Directory counts should match") - + // Expected counts based on test structure assert.Equal(t, 4, originalStats.FileCount, "Should count 4 files") assert.Equal(t, 4, originalStats.DirCount, "Should count 4 directories (level1, level1/level2, empty_dir, level1/empty_subdir)") }) - + // Test 2: Incomplete backup - should fail validation t.Run("Incomplete Backup Fails Validation", func(t *testing.T) { err := createIncompleteBackup(originalDir, backupFile+"_incomplete") require.NoError(t, err) - + originalStats, err := countServerFiles(originalDir) require.NoError(t, err) - + backupStats, err := countBackupEntries(backupFile + "_incomplete") require.NoError(t, err) - + // Incomplete backup should have fewer entries assert.Less(t, backupStats.FileCount, originalStats.FileCount, "Incomplete backup should have fewer files") }) @@ -86,46 +86,46 @@ func TestContentValidationFileCount(t *testing.T) { // countServerFiles simulates Server.countServerFilesAndDirs for testing func countServerFiles(serverPath string) (*fileStats, error) { stats := &fileStats{} - + err := filepath.Walk(serverPath, func(path string, info os.FileInfo, err error) error { if err != nil { return nil // Skip unreadable files } - + // Skip root directory if path == serverPath { return nil } - + if info.IsDir() { stats.DirCount++ } else { stats.FileCount++ } - + return nil }) - + return stats, err } // countBackupEntries simulates Server.countBackupEntries for testing func countBackupEntries(backupPath string) (*fileStats, error) { stats := &fileStats{} - + f, err := os.Open(backupPath) if err != nil { return nil, err } defer f.Close() - + // Simple GZIP reader for test gr, err := gzip.NewReader(f) if err != nil { return nil, err } defer gr.Close() - + // Scan TAR headers tarReader := tar.NewReader(gr) for { @@ -136,20 +136,20 @@ func countBackupEntries(backupPath string) (*fileStats, error) { if err != nil { return nil, err } - + // Skip root entries (same logic as real implementation) - if header.Name == "." || header.Name == "" || header.Name == "/" || - header.Name == "./" || strings.HasPrefix(header.Name, "../") { + if header.Name == "." || header.Name == "" || header.Name == "/" || + header.Name == "./" || strings.HasPrefix(header.Name, "../") { continue } - + if header.FileInfo().IsDir() { stats.DirCount++ } else { stats.FileCount++ } } - + return stats, nil } @@ -166,39 +166,39 @@ func createCorrectBackup(sourceDir, backupFile string) error { return err } defer f.Close() - + gw := gzip.NewWriter(f) defer gw.Close() - + tw := tar.NewWriter(gw) defer tw.Close() - + // Walk and archive everything (including directories) return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - + if path == sourceDir { return nil // Skip root } - + rel, err := filepath.Rel(sourceDir, path) if err != nil { return err } - + // Create TAR header header, err := tar.FileInfoHeader(info, "") if err != nil { return err } header.Name = rel - + if err := tw.WriteHeader(header); err != nil { return err } - + // Write file content for regular files if !info.IsDir() { file, err := os.Open(path) @@ -206,13 +206,13 @@ func createCorrectBackup(sourceDir, backupFile string) error { return err } defer file.Close() - + _, err = io.Copy(tw, file) if err != nil { return err } } - + return nil }) } @@ -224,56 +224,56 @@ func createIncompleteBackup(sourceDir, backupFile string) error { return err } defer f.Close() - + gw := gzip.NewWriter(f) defer gw.Close() - + tw := tar.NewWriter(gw) defer tw.Close() - + // Only backup some files (skip files containing "level2") return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - + if path == sourceDir { return nil } - + rel, err := filepath.Rel(sourceDir, path) if err != nil { return err } - + // Skip level2 entries to simulate incomplete backup if strings.Contains(rel, "level2") { return nil } - + header, err := tar.FileInfoHeader(info, "") if err != nil { return err } header.Name = rel - + if err := tw.WriteHeader(header); err != nil { return err } - + if !info.IsDir() { file, err := os.Open(path) if err != nil { return err } defer file.Close() - + _, err = io.Copy(tw, file) if err != nil { return err } } - + return nil }) -} \ No newline at end of file +} diff --git a/server/backup/download_progress.go b/server/backup/download_progress.go index df6be8355..e302455d0 100644 --- a/server/backup/download_progress.go +++ b/server/backup/download_progress.go @@ -10,13 +10,13 @@ import ( // DownloadProgressReader wraps an io.ReadCloser to track download progress type DownloadProgressReader struct { - reader io.ReadCloser - totalSize int64 - downloaded atomic.Int64 - lastReport atomic.Int64 - onProgress func(downloaded, total int64) - logger *log.Entry - backupID string + reader io.ReadCloser + totalSize int64 + downloaded atomic.Int64 + lastReport atomic.Int64 + onProgress func(downloaded, total int64) + logger *log.Entry + backupID string } // NewDownloadProgressReader creates a new progress-tracking reader @@ -36,7 +36,7 @@ func (r *DownloadProgressReader) Read(p []byte) (n int, err error) { if n > 0 { // Update downloaded bytes current := r.downloaded.Add(int64(n)) - + // Report progress at most once per 100ms to avoid spam now := time.Now().UnixNano() lastReport := r.lastReport.Load() @@ -45,7 +45,7 @@ func (r *DownloadProgressReader) Read(p []byte) (n int, err error) { if r.onProgress != nil { r.onProgress(current, r.totalSize) } - + // Log progress every 10% if r.totalSize > 0 { percentage := (current * 100) / r.totalSize @@ -60,7 +60,7 @@ func (r *DownloadProgressReader) Read(p []byte) (n int, err error) { } } } - + // Log completion if err == io.EOF && r.totalSize > 0 { r.logger.WithFields(log.Fields{ @@ -68,11 +68,11 @@ func (r *DownloadProgressReader) Read(p []byte) (n int, err error) { "total_mb": r.totalSize / (1024 * 1024), }).Info("S3 download completed") } - + return n, err } // Close implements io.Closer func (r *DownloadProgressReader) Close() error { return r.reader.Close() -} \ No newline at end of file +} diff --git a/server/backup/locate_local_test.go b/server/backup/locate_local_test.go new file mode 100644 index 000000000..c0fe59046 --- /dev/null +++ b/server/backup/locate_local_test.go @@ -0,0 +1,47 @@ +package backup + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/Rene-Roscher/wings/config" +) + +// A legacy uncompressed .tar backup found by LocateLocal must be the file +// that Size, Checksum and Details read, not the default .tar.gz path. +func TestLocateLocalTarBackupDetails(t *testing.T) { + backupDir := t.TempDir() + config.Set(&config.Configuration{ + AuthenticationToken: "test-token", + System: config.SystemConfiguration{ + BackupDirectory: backupDir, + }, + }) + + uuid := "11111111-1111-1111-1111-111111111111" + tarPath := filepath.Join(backupDir, uuid+".tar") + if err := os.WriteFile(tarPath, []byte("archive"), 0o600); err != nil { + t.Fatal(err) + } + + b, st, err := LocateLocal(nil, uuid) + if err != nil { + t.Fatalf("LocateLocal: %v", err) + } + if b.Path() != tarPath { + t.Fatalf("expected path %q, got %q", tarPath, b.Path()) + } + if st.Size() != 7 { + t.Fatalf("expected stat size 7, got %d", st.Size()) + } + + ad, err := b.Details(context.Background(), nil) + if err != nil { + t.Fatalf("Details: %v", err) + } + if ad.Size != 7 || ad.ChecksumType != "sha256" || ad.Checksum == "" { + t.Fatalf("unexpected details: %+v", ad) + } +} diff --git a/server/backup/restore_edge_case_test.go b/server/backup/restore_edge_case_test.go index 61510a34a..ddfe0b8d2 100644 --- a/server/backup/restore_edge_case_test.go +++ b/server/backup/restore_edge_case_test.go @@ -20,32 +20,32 @@ func TestRestoreHandlesRootDirectory(t *testing.T) { tempDir := t.TempDir() backupFile := filepath.Join(tempDir, "test-with-root.tar.gz") restoreDir := filepath.Join(tempDir, "restore") - + require.NoError(t, os.MkdirAll(restoreDir, 0755)) - + // Create backup that includes root directory entry "." t.Log("Creating backup with root directory entry...") err := createBackupWithRootEntry(backupFile) require.NoError(t, err) - + // Test restore callback handles "." correctly var handledEntries []string var errorCount int - + err = restoreTestBackup(backupFile, func(file string, info os.FileInfo, r io.ReadCloser) error { defer r.Close() - + t.Logf("Processing entry: '%s' (isDir: %v, size: %d)", file, info.IsDir(), info.Size()) handledEntries = append(handledEntries, file) - + // Skip root directory entries - this is the fix for the original error if file == "." || file == "" { t.Logf("Skipping root directory entry: '%s'", file) return nil } - + fullPath := filepath.Join(restoreDir, file) - + if info.IsDir() { // Fixed logic: handle directories correctly if err := os.MkdirAll(fullPath, info.Mode()); err != nil { @@ -59,12 +59,12 @@ func TestRestoreHandlesRootDirectory(t *testing.T) { if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { return err } - + content, err := io.ReadAll(r) if err != nil { return err } - + if err := os.WriteFile(fullPath, content, info.Mode()); err != nil { t.Logf("Error writing file %s: %v", fullPath, err) errorCount++ @@ -72,21 +72,21 @@ func TestRestoreHandlesRootDirectory(t *testing.T) { } t.Logf("Created file: %s (%d bytes)", file, len(content)) } - + return nil }) - + require.NoError(t, err, "Restore should complete without errors") assert.Equal(t, 0, errorCount, "Should have no processing errors") assert.Greater(t, len(handledEntries), 0, "Should have processed some entries") - + // Verify that if "." was in the archive, it was handled gracefully for _, entry := range handledEntries { if entry == "." { t.Log("Root directory entry '.' was handled correctly (skipped)") } } - + t.Logf("SUCCESS: Processed %d entries with %d errors", len(handledEntries), errorCount) } @@ -97,13 +97,13 @@ func createBackupWithRootEntry(backupFile string) error { return err } defer f.Close() - + gw := gzip.NewWriter(f) defer gw.Close() - + tw := tar.NewWriter(gw) defer tw.Close() - + // Add a potentially problematic root directory entry rootHeader := &tar.Header{ Name: ".", @@ -111,11 +111,11 @@ func createBackupWithRootEntry(backupFile string) error { Typeflag: tar.TypeDir, ModTime: time.Now(), } - + if err := tw.WriteHeader(rootHeader); err != nil { return err } - + // Add some regular entries entries := []struct { name string @@ -126,10 +126,10 @@ func createBackupWithRootEntry(backupFile string) error { {"test_file.txt", "test content", false}, {"test_dir/nested_file.txt", "nested content", false}, } - + for _, entry := range entries { var header *tar.Header - + if entry.isDir { header = &tar.Header{ Name: entry.name, @@ -146,18 +146,18 @@ func createBackupWithRootEntry(backupFile string) error { ModTime: time.Now(), } } - + if err := tw.WriteHeader(header); err != nil { return err } - + if !entry.isDir { if _, err := tw.Write([]byte(entry.content)); err != nil { return err } } } - + return nil } @@ -166,7 +166,7 @@ func TestRestoreSkipsInvalidPaths(t *testing.T) { tempDir := t.TempDir() restoreDir := filepath.Join(tempDir, "restore") require.NoError(t, os.MkdirAll(restoreDir, 0755)) - + // Test various problematic paths problematicPaths := []struct { path string @@ -181,11 +181,11 @@ func TestRestoreSkipsInvalidPaths(t *testing.T) { {"normal_file.txt", false, "normal file"}, {"normal_dir", false, "normal directory"}, } - + for _, test := range problematicPaths { t.Run(test.description, func(t *testing.T) { var wasSkipped bool - + // This simulates the fix we applied to the restore logic if test.path == "." || test.path == "" || test.path == "/" || test.path == "./" || strings.HasPrefix(test.path, "../") { wasSkipped = true @@ -194,9 +194,9 @@ func TestRestoreSkipsInvalidPaths(t *testing.T) { wasSkipped = false t.Logf("Processing valid path: '%s'", test.path) } - - assert.Equal(t, test.shouldSkip, wasSkipped, + + assert.Equal(t, test.shouldSkip, wasSkipped, "Path '%s' skip behavior should match expected", test.path) }) } -} \ No newline at end of file +} diff --git a/server/backup/restore_test.go b/server/backup/restore_test.go index af268006a..4ab04bda4 100644 --- a/server/backup/restore_test.go +++ b/server/backup/restore_test.go @@ -20,28 +20,28 @@ func TestLocalBackupRestoreCycle(t *testing.T) { backupFile := filepath.Join(tempDir, "test-backup.tar.gz") originalDir := filepath.Join(tempDir, "original") restoreDir := filepath.Join(tempDir, "restored") - + require.NoError(t, os.MkdirAll(originalDir, 0755)) require.NoError(t, os.MkdirAll(restoreDir, 0755)) - + // Create test structure with files and empty directories testStructure := map[string]string{ - "file1.txt": "content1", - "file2.txt": "content2", - "level1/file3.txt": "content3", - "level1/level2/file4.txt": "content4", + "file1.txt": "content1", + "file2.txt": "content2", + "level1/file3.txt": "content3", + "level1/level2/file4.txt": "content4", // CRITICAL: Empty directories that MUST be preserved - "empty_dir/.keep": "", - "level1/empty_subdir/.keep": "", - "level1/level2/empty_deep/.keep": "", + "empty_dir/.keep": "", + "level1/empty_subdir/.keep": "", + "level1/level2/empty_deep/.keep": "", "completely_empty_chain/level1/level2/level3/.keep": "", } - + // Create original test files and directories for path, content := range testStructure { fullPath := filepath.Join(originalDir, path) require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) - + if strings.HasSuffix(path, ".keep") { // Create empty directory marker and remove it to leave empty directory require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) @@ -50,11 +50,11 @@ func TestLocalBackupRestoreCycle(t *testing.T) { require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) } } - + // List all original directories for comparison originalDirs := make(map[string]bool) originalFiles := make(map[string]string) - + err := filepath.WalkDir(originalDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err @@ -63,7 +63,7 @@ func TestLocalBackupRestoreCycle(t *testing.T) { if relPath == "." { return nil } - + if d.IsDir() { originalDirs[relPath] = true } else { @@ -76,35 +76,35 @@ func TestLocalBackupRestoreCycle(t *testing.T) { return nil }) require.NoError(t, err) - + t.Logf("Original structure: %d directories, %d files", len(originalDirs), len(originalFiles)) - + // Create backup manually using TAR+GZIP (simulating fixed Wings backup logic) t.Log("Creating backup archive...") err = createTestBackup(originalDir, backupFile) require.NoError(t, err) assert.FileExists(t, backupFile) - + // Verify archive contents t.Log("Verifying archive contents...") archiveContents, err := listTarArchive(backupFile) require.NoError(t, err) - + t.Logf("Archive contains %d entries", len(archiveContents)) for _, entry := range archiveContents { t.Logf("Archive entry: %s (isDir: %v)", entry.Name, entry.IsDir) } - + // Restore using our fixed callback logic t.Log("Testing restore logic...") restoredDirs := make(map[string]bool) restoredFiles := make(map[string]string) - + err = restoreTestBackup(backupFile, func(file string, info os.FileInfo, r io.ReadCloser) error { defer r.Close() - + fullPath := filepath.Join(restoreDir, file) - + if info.IsDir() { // Test our fixed directory handling logic restoredDirs[file] = true @@ -115,49 +115,49 @@ func TestLocalBackupRestoreCycle(t *testing.T) { if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { return err } - + content, err := io.ReadAll(r) if err != nil { return err } restoredFiles[file] = string(content) - + return os.WriteFile(fullPath, content, info.Mode()) } }) require.NoError(t, err) - + // Verify all directories were restored t.Log("Verifying directory restoration...") for dir := range originalDirs { assert.True(t, restoredDirs[dir], "Directory %s should be restored", dir) assert.DirExists(t, filepath.Join(restoreDir, dir), "Directory %s should exist after restore", dir) } - + // Verify all files were restored with correct content t.Log("Verifying file restoration...") for file, originalContent := range originalFiles { assert.Equal(t, originalContent, restoredFiles[file], "File content should match for %s", file) assert.FileExists(t, filepath.Join(restoreDir, file), "File %s should exist after restore", file) } - + // Verify critical empty directories are preserved criticalEmptyDirs := []string{ "empty_dir", - "level1/empty_subdir", + "level1/empty_subdir", "level1/level2/empty_deep", "completely_empty_chain", "completely_empty_chain/level1", - "completely_empty_chain/level1/level2", + "completely_empty_chain/level1/level2", "completely_empty_chain/level1/level2/level3", } - + t.Log("Verifying critical empty directories...") for _, dir := range criticalEmptyDirs { assert.True(t, restoredDirs[dir], "Critical empty directory %s MUST be restored", dir) assert.DirExists(t, filepath.Join(restoreDir, dir), "Critical empty directory %s MUST exist after restore", dir) } - + t.Logf("SUCCESS: Restored %d directories and %d files", len(restoredDirs), len(restoredFiles)) } @@ -175,28 +175,28 @@ func createTestBackup(sourceDir, backupFile string) error { return err } defer f.Close() - + gw := gzip.NewWriter(f) defer gw.Close() - + tw := tar.NewWriter(gw) defer tw.Close() - + return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - + // Skip root directory if path == sourceDir { return nil } - + rel, err := filepath.Rel(sourceDir, path) if err != nil { return err } - + // Create TAR header header, err := tar.FileInfoHeader(info, "") if err != nil { @@ -204,12 +204,12 @@ func createTestBackup(sourceDir, backupFile string) error { } header.Name = rel header.ModTime = info.ModTime() - + // Write header (CRITICAL: includes directories!) if err := tw.WriteHeader(header); err != nil { return err } - + // Write file content for regular files only if !info.IsDir() { file, err := os.Open(path) @@ -217,13 +217,13 @@ func createTestBackup(sourceDir, backupFile string) error { return err } defer file.Close() - + _, err = io.Copy(tw, file) if err != nil { return err } } - + return nil }) } @@ -235,15 +235,15 @@ func listTarArchive(backupFile string) ([]archiveEntry, error) { return nil, err } defer f.Close() - + gr, err := gzip.NewReader(f) if err != nil { return nil, err } defer gr.Close() - + tr := tar.NewReader(gr) - + var entries []archiveEntry for { header, err := tr.Next() @@ -253,14 +253,14 @@ func listTarArchive(backupFile string) ([]archiveEntry, error) { if err != nil { return nil, err } - + entries = append(entries, archiveEntry{ Name: header.Name, IsDir: header.FileInfo().IsDir(), Size: header.Size, }) } - + return entries, nil } @@ -271,15 +271,15 @@ func restoreTestBackup(backupFile string, callback func(string, os.FileInfo, io. return err } defer f.Close() - + gr, err := gzip.NewReader(f) if err != nil { return err } defer gr.Close() - + tr := tar.NewReader(gr) - + for { header, err := tr.Next() if err == io.EOF { @@ -288,7 +288,7 @@ func restoreTestBackup(backupFile string, callback func(string, os.FileInfo, io. if err != nil { return err } - + // Create a ReadCloser from the tar reader for this entry var r io.ReadCloser if header.FileInfo().IsDir() { @@ -298,7 +298,7 @@ func restoreTestBackup(backupFile string, callback func(string, os.FileInfo, io. // For files, provide limited reader with file content r = io.NopCloser(io.LimitReader(tr, header.Size)) } - + // Call the callback with simulated file info info := &testFileInfo{ name: filepath.Base(header.Name), @@ -307,12 +307,12 @@ func restoreTestBackup(backupFile string, callback func(string, os.FileInfo, io. modTime: header.ModTime, isDir: header.FileInfo().IsDir(), } - + if err := callback(header.Name, info, r); err != nil { return err } } - + return nil } @@ -330,4 +330,4 @@ func (fi *testFileInfo) Size() int64 { return fi.size } func (fi *testFileInfo) Mode() os.FileMode { return fi.mode } func (fi *testFileInfo) ModTime() time.Time { return fi.modTime } func (fi *testFileInfo) IsDir() bool { return fi.isDir } -func (fi *testFileInfo) Sys() interface{} { return nil } \ No newline at end of file +func (fi *testFileInfo) Sys() interface{} { return nil } diff --git a/server/backup/s3_simple_test.go b/server/backup/s3_simple_test.go index 4c6861a2a..0c2d1008c 100644 --- a/server/backup/s3_simple_test.go +++ b/server/backup/s3_simple_test.go @@ -35,24 +35,24 @@ func TestS3RestoreDirectoryHandling(t *testing.T) { // Create test TAR data in memory (S3 Restore expects ALREADY DECOMPRESSED TAR stream) var buf bytes.Buffer tw := tar.NewWriter(&buf) - + // Create test entries including the problematic cases entries := []struct { name string content string isDir bool }{ - {".", "", true}, // Root directory (causes the original error) - {"empty_dir", "", true}, // Empty directory - {"test_dir", "", true}, // Regular directory - {"file1.txt", "content1", false}, // Regular file + {".", "", true}, // Root directory (causes the original error) + {"empty_dir", "", true}, // Empty directory + {"test_dir", "", true}, // Regular directory + {"file1.txt", "content1", false}, // Regular file {"test_dir/file2.txt", "content2", false}, // Nested file } - + // Write entries to TAR for _, entry := range entries { var header *tar.Header - + if entry.isDir { header = &tar.Header{ Name: entry.name, @@ -69,9 +69,9 @@ func TestS3RestoreDirectoryHandling(t *testing.T) { ModTime: time.Now(), } } - + require.NoError(t, tw.WriteHeader(header)) - + if !entry.isDir { _, err := tw.Write([]byte(entry.content)) require.NoError(t, err) @@ -87,26 +87,26 @@ func TestS3RestoreDirectoryHandling(t *testing.T) { adapter: S3BackupAdapter, }, } - + // Test restore with our fixed logic ctx := context.Background() restoreReader := bytes.NewReader(buf.Bytes()) - + var processedEntries []string var errorCount int - + err := s3backup.Restore(ctx, restoreReader, func(file string, info os.FileInfo, r io.ReadCloser) error { defer r.Close() - + t.Logf("S3 processing entry: '%s' (isDir: %v, size: %d)", file, info.IsDir(), info.Size()) processedEntries = append(processedEntries, file) - + // Simulate the same logic as in server/backup.go restore callback if file == "." || file == "" || file == "/" { t.Logf("Correctly skipped problematic entry: '%s'", file) return nil } - + if info.IsDir() { t.Logf("Would create directory: %s", file) } else { @@ -117,23 +117,23 @@ func TestS3RestoreDirectoryHandling(t *testing.T) { } t.Logf("Would write file: %s (%d bytes)", file, len(content)) } - + return nil }) - + require.NoError(t, err, "S3 restore should complete without errors") assert.Equal(t, 0, errorCount, "Should have no processing errors") - + // Verify all entries were processed assert.Len(t, processedEntries, 5, "Should process all 5 entries") - + // Verify specific entries assert.Contains(t, processedEntries, ".", "Root directory should be processed (but skipped)") assert.Contains(t, processedEntries, "empty_dir", "Empty directory should be processed") assert.Contains(t, processedEntries, "test_dir", "Test directory should be processed") assert.Contains(t, processedEntries, "file1.txt", "File1 should be processed") assert.Contains(t, processedEntries, "test_dir/file2.txt", "Nested file should be processed") - + t.Logf("SUCCESS: S3 restore processed %d entries correctly", len(processedEntries)) } @@ -144,7 +144,7 @@ func TestS3RestoreCompressionDetection(t *testing.T) { // Create simple TAR (decompression happens before S3 Restore is called) var buf bytes.Buffer tw := tar.NewWriter(&buf) - + // Single test entry header := &tar.Header{ Name: "test.txt", @@ -153,7 +153,7 @@ func TestS3RestoreCompressionDetection(t *testing.T) { Typeflag: tar.TypeReg, ModTime: time.Now(), } - + require.NoError(t, tw.WriteHeader(header)) _, err := tw.Write([]byte("test")) require.NoError(t, err) @@ -166,15 +166,15 @@ func TestS3RestoreCompressionDetection(t *testing.T) { adapter: S3BackupAdapter, }, } - + ctx := context.Background() restoreReader := bytes.NewReader(buf.Bytes()) - + var detectedContent string - + err = s3backup.Restore(ctx, restoreReader, func(file string, info os.FileInfo, r io.ReadCloser) error { defer r.Close() - + if strings.HasSuffix(file, ".txt") { content, err := io.ReadAll(r) if err != nil { @@ -182,13 +182,13 @@ func TestS3RestoreCompressionDetection(t *testing.T) { } detectedContent = string(content) } - + return nil }) - + require.NoError(t, err, "S3 restore should handle TAR extraction") assert.Equal(t, "test", detectedContent, "Content should be correctly extracted") t.Log("SUCCESS: S3 restore correctly processed TAR archive") }) -} \ No newline at end of file +} diff --git a/server/backup_cleanup_test.go b/server/backup_cleanup_test.go index 21cb71d4c..8b2f9734a 100644 --- a/server/backup_cleanup_test.go +++ b/server/backup_cleanup_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/Rene-Roscher/wings/config" - "github.com/Rene-Roscher/wings/server/backup" ) func init() { @@ -31,7 +30,6 @@ func TestBackupCleanupOnServerDeletion(t *testing.T) { // Test server deletion cleanup for multiple servers servers := []string{"server1", "server2", "server3"} - var allOperations []string // Register operations for different servers (limit to stay under 8 total) for _, serverID := range servers { @@ -39,8 +37,7 @@ func TestBackupCleanupOnServerDeletion(t *testing.T) { backupID := serverID + "_backup_" + string(rune(i+'0')) _, _, cancel, err, _ := registry.Register(ctx, backupID, serverID, OperationTypeBackup) require.NoError(t, err) - allOperations = append(allOperations, backupID) - + // Clean up individual operations defer func(bid string) { cancel() @@ -113,22 +110,6 @@ func TestBackupCleanupStaleOperations(t *testing.T) { assert.Equal(t, initialCount, registry.Count()) } -// TestLocalBackupCleanupFunction tests the local backup file cleanup -func TestLocalBackupCleanupFunction(t *testing.T) { - // This test verifies the cleanup function exists and can be called - // We don't test actual file operations to avoid affecting real files - - // Test cleanup function doesn't panic with non-existent server - err := backup.CleanupBackupFilesForServer("non-existent-server-123") - - // Function should handle non-existent servers gracefully - // Error is acceptable (directory not found), panic is not - if err != nil { - assert.Contains(t, err.Error(), "no such file or directory", - "Should handle non-existent server gracefully") - } -} - // TestBackupCleanupRace tests concurrent cleanup operations func TestBackupCleanupRace(t *testing.T) { registry := GetBackupOperationRegistry() @@ -150,7 +131,7 @@ func TestBackupCleanupRace(t *testing.T) { // Cleanup by server go func() { - registry.CancelAllForServer("race_server") + _ = registry.CancelAllForServer("race_server") done <- true }() @@ -166,7 +147,7 @@ func TestBackupCleanupRace(t *testing.T) { // Should not have any operations for race_server assert.Equal(t, 0, registry.CountForServer("race_server")) - + // Registry should be in consistent state finalCount := registry.Count() assert.True(t, finalCount <= initialCount, "Count should not increase") @@ -228,12 +209,12 @@ func TestServerDeletionBackupIntegration(t *testing.T) { // Verify operations are active assert.Equal(t, 2, registry.CountForServer(serverID)) - + retrieved, exists := registry.Get("backup_op") assert.True(t, exists) assert.Equal(t, backupOp.ID, retrieved.ID) - retrieved, exists = registry.Get("restore_op") + retrieved, exists = registry.Get("restore_op") assert.True(t, exists) assert.Equal(t, restoreOp.ID, retrieved.ID) @@ -243,14 +224,14 @@ func TestServerDeletionBackupIntegration(t *testing.T) { // Verify all operations are cancelled and removed assert.Equal(t, 0, registry.CountForServer(serverID)) - + _, exists = registry.Get("backup_op") assert.False(t, exists, "Backup operation should be removed") - + _, exists = registry.Get("restore_op") assert.False(t, exists, "Restore operation should be removed") // Cleanup backupCancel() restoreCancel() -} \ No newline at end of file +} diff --git a/server/backup_operations.go b/server/backup_operations.go index cccca91f0..b58a64341 100644 --- a/server/backup_operations.go +++ b/server/backup_operations.go @@ -50,7 +50,7 @@ type BackupOperationRegistry struct { // CRITICAL: Operation limits to prevent resource exhaustion with queuing support maxConcurrentBackups int maxConcurrentRestores int - // QUEUING: Semaphores to handle waiting instead of immediate rejection + // QUEUING: Semaphores to handle waiting instead of immediate rejection backupSemaphore chan struct{} restoreSemaphore chan struct{} } @@ -59,7 +59,7 @@ type BackupOperationRegistry struct { func NewBackupOperationRegistry() *BackupOperationRegistry { maxBackups := 8 maxRestores := 8 - + return &BackupOperationRegistry{ operations: make(map[string]*BackupOperation), logger: log.WithField("component", "backup_registry"), @@ -89,13 +89,13 @@ func (r *BackupOperationRegistry) Register(parentCtx context.Context, backupID, // ATOMIC: Check if we need to wait (for accurate state reporting) needsQueue := len(semaphore) >= cap(semaphore) - + // WAIT in queue until slot available or context cancelled select { case semaphore <- struct{}{}: // Successfully acquired slot r.logger.WithFields(log.Fields{ - "backup_id": backupID, - "type": opType, + "backup_id": backupID, + "type": opType, "was_queued": needsQueue, }).Debug("acquired operation slot from queue") case <-parentCtx.Done(): @@ -124,10 +124,10 @@ func (r *BackupOperationRegistry) Register(parentCtx context.Context, backupID, // Check if operation already exists (shouldn't happen with proper state management) if existing, exists := r.operations[backupID]; exists { r.logger.WithFields(log.Fields{ - "backup_id": backupID, - "existing_id": existing.ID, - "new_id": operationID, - "type": opType, + "backup_id": backupID, + "existing_id": existing.ID, + "new_id": operationID, + "type": opType, }).Warn("backup operation already exists, replacing") } @@ -235,7 +235,7 @@ func (r *BackupOperationRegistry) Count() int { func (r *BackupOperationRegistry) GetQueueStatus() map[string]any { r.mu.RLock() defer r.mu.RUnlock() - + backupCount := 0 restoreCount := 0 for _, op := range r.operations { @@ -246,18 +246,18 @@ func (r *BackupOperationRegistry) GetQueueStatus() map[string]any { restoreCount++ } } - + return map[string]any{ "backups": map[string]any{ - "active": backupCount, - "max": r.maxConcurrentBackups, - "available": r.maxConcurrentBackups - backupCount, + "active": backupCount, + "max": r.maxConcurrentBackups, + "available": r.maxConcurrentBackups - backupCount, "queue_length": len(r.backupSemaphore), }, "restores": map[string]any{ - "active": restoreCount, - "max": r.maxConcurrentRestores, - "available": r.maxConcurrentRestores - restoreCount, + "active": restoreCount, + "max": r.maxConcurrentRestores, + "available": r.maxConcurrentRestores - restoreCount, "queue_length": len(r.restoreSemaphore), }, "total_operations": len(r.operations), @@ -325,7 +325,7 @@ func (r *BackupOperationRegistry) CancelAllForServer(serverID string) error { defer r.mu.Unlock() var cancelledOps []string - + for backupID, operation := range r.operations { if operation.ServerID == serverID { r.logger.WithFields(log.Fields{ @@ -337,13 +337,13 @@ func (r *BackupOperationRegistry) CancelAllForServer(serverID string) error { // Cancel the context operation.Cancel() - + // CRITICAL FIX: Release semaphore token BLOCKING (prevents leaks) if operation.semaphoreToken != nil { <-operation.semaphoreToken // BLOCKING receive to return token r.logger.Debug("released semaphore slot for server deletion") } - + // Remove from registry delete(r.operations, backupID) cancelledOps = append(cancelledOps, backupID) @@ -352,9 +352,9 @@ func (r *BackupOperationRegistry) CancelAllForServer(serverID string) error { if len(cancelledOps) > 0 { r.logger.WithFields(log.Fields{ - "server_id": serverID, - "cancelled_count": len(cancelledOps), - "cancelled_ops": cancelledOps, + "server_id": serverID, + "cancelled_count": len(cancelledOps), + "cancelled_ops": cancelledOps, }).Info("cancelled all backup operations for server deletion") } diff --git a/server/backup_progress.go b/server/backup_progress.go index 187bf2b2a..a95746186 100644 --- a/server/backup_progress.go +++ b/server/backup_progress.go @@ -6,26 +6,26 @@ import ( "sync/atomic" "time" - "github.com/apex/log" "github.com/Rene-Roscher/wings/internal/progress" + "github.com/apex/log" ) // SimpleProgressTracker - ultra-lightweight progress tracking with ZERO overhead type SimpleProgressTracker struct { - server *Server - backupID string - backupType string - progress *progress.Progress - lastSent int64 // Last percentage sent - lastTime int64 // Last time sent (nanoseconds) - lastBytes int64 // Last bytes value sent (for detecting changes at 100%) - isS3 bool // Whether this is an S3 backup (needs 80/20 split) - archiveSize int64 // Size of archive (for S3 80/20 calculation) - + server *Server + backupID string + backupType string + progress *progress.Progress + lastSent int64 // Last percentage sent + lastTime int64 // Last time sent (nanoseconds) + lastBytes int64 // Last bytes value sent (for detecting changes at 100%) + isS3 bool // Whether this is an S3 backup (needs 80/20 split) + archiveSize int64 // Size of archive (for S3 80/20 calculation) + // Context-aware goroutine management - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup } // BackupProgressUpdate represents the data sent over WebSocket @@ -56,7 +56,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { lastSent := atomic.LoadInt64(&spt.lastSent) // SINGLE THROTTLING CHECK: Only send events maximum every 250ms to prevent WebSocket flooding - const throttleIntervalNanos = 250_000_000 // 250ms in nanoseconds + const throttleIntervalNanos = 250_000_000 // 250ms in nanoseconds shouldSendByTime := (now - lastTime) >= throttleIntervalNanos // Check if this is initial progress (0%) @@ -65,7 +65,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { if total > 0 { // Standard percentage calculation first rawPercentage := int((written * 100) / total) - + // S3 SPECIAL CASE: Scale to 80% during archive, then 80-100% during upload if spt.isS3 { if spt.archiveSize == 0 { @@ -99,7 +99,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { percentage = rawPercentage } percentage = min(100, percentage) - + // Send on percentage increase AND time throttle (OR initial) percentageChanged := percentage > int(lastSent) // Also send if bytes changed (for updates within same percentage) @@ -111,10 +111,10 @@ func (spt *SimpleProgressTracker) CheckProgress() { atomic.StoreInt64(&spt.lastBytes, int64(written)) } } else { - // Byte mode - show progress in 1MB chunks with time throttling + // Byte mode - show progress in 1MB chunks with time throttling percentage = 0 // Use 0% for unknown total instead of -1 lastMB := lastSent - currentMB := written / (1024 * 1024) // 1MB chunks + currentMB := written / (1024 * 1024) // 1MB chunks dataChanged := currentMB > lastMB || written > 0 // Include any progress shouldSend = (dataChanged && shouldSendByTime) || isInitialProgress if shouldSend { @@ -124,7 +124,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { // ALWAYS send initial progress (0%) and final progress (100%) - but only ONCE! isFinalProgress := total > 0 && percentage >= 100 && atomic.LoadInt64(&spt.lastSent) < 100 - + if shouldSend || isFinalProgress { atomic.StoreInt64(&spt.lastTime, now) if percentage >= 0 { @@ -139,7 +139,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { default: } } - + // CRITICAL FIX: Send events SYNCHRONOUSLY during normal progress // Only the FINAL events truly need to be async to avoid blocking restore completion // Regular progress events are fast enough to send inline @@ -163,7 +163,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { return } }() - + // Final check before send if spt.ctx != nil { select { @@ -172,7 +172,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { default: } } - + spt.server.Events().Publish(BackupProgressEvent, update) spt.server.Log().WithField("backup_id", spt.backupID). WithField("percentage", percentage). @@ -183,7 +183,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { } else { // Send normal progress events synchronously - they're fast! spt.server.Events().Publish(BackupProgressEvent, update) - + // Log initial event for debugging if isInitialProgress { spt.server.Log().WithField("backup_id", spt.backupID). @@ -197,7 +197,7 @@ func (spt *SimpleProgressTracker) CheckProgress() { // NewSimpleProgressTracker creates a progress tracker with proper lifecycle management func NewSimpleProgressTracker(ctx context.Context, server *Server, backupID, backupType string, progress *progress.Progress) *SimpleProgressTracker { - progCtx, cancel := context.WithCancel(ctx) + progCtx, cancel := context.WithCancel(ctx) //nolint:gosec // cancel is stored on the tracker and called by Close() return &SimpleProgressTracker{ server: server, backupID: backupID, @@ -222,7 +222,7 @@ func (spt *SimpleProgressTracker) Close() { spt.cancel() spt.cancel = nil // Prevent double-cancel } - + // Wait for any pending CheckProgress goroutines with SHORT timeout // These are fire-and-forget event sends, we don't need to wait long done := make(chan struct{}) @@ -230,7 +230,7 @@ func (spt *SimpleProgressTracker) Close() { spt.wg.Wait() close(done) }() - + select { case <-done: // All goroutines finished cleanly @@ -258,14 +258,14 @@ func (spt *SimpleProgressTracker) SendFinalProgress(success bool) { } spt.server.Log().WithFields(log.Fields{ - "backup_id": spt.backupID, - "backup_type": spt.backupType, - "success": success, - "percentage": percentage, - "written": written, - "total": total, - "is_restoring": spt.server.IsRestoring(), - "server_state": spt.server.Environment.State(), + "backup_id": spt.backupID, + "backup_type": spt.backupType, + "success": success, + "percentage": percentage, + "written": written, + "total": total, + "is_restoring": spt.server.IsRestoring(), + "server_state": spt.server.Environment.State(), }).Info("SENDING FINAL PROGRESS EVENT") update := BackupProgressUpdate{ @@ -286,7 +286,7 @@ func (spt *SimpleProgressTracker) SendFinalProgress(success bool) { default: } } - + // Send the final event directly - this is fast enough spt.server.Events().Publish(BackupProgressEvent, update) diff --git a/server/backup_progress_validation_test.go b/server/backup_progress_validation_test.go index 12da4f78a..8fa4ea5da 100644 --- a/server/backup_progress_validation_test.go +++ b/server/backup_progress_validation_test.go @@ -17,28 +17,30 @@ import ( // mockEnvironment is a minimal mock implementation for testing type mockEnvironment struct{} -func (m *mockEnvironment) Type() string { return "mock" } -func (m *mockEnvironment) Config() *environment.Configuration { return nil } -func (m *mockEnvironment) Events() *events.Bus { return events.NewBus() } -func (m *mockEnvironment) Exists() (bool, error) { return true, nil } +func (m *mockEnvironment) Type() string { return "mock" } +func (m *mockEnvironment) Config() *environment.Configuration { return nil } +func (m *mockEnvironment) Events() *events.Bus { return events.NewBus() } +func (m *mockEnvironment) Exists() (bool, error) { return true, nil } func (m *mockEnvironment) IsRunning(ctx context.Context) (bool, error) { return false, nil } -func (m *mockEnvironment) InSituUpdate() error { return nil } -func (m *mockEnvironment) OnBeforeStart(ctx context.Context) error { return nil } -func (m *mockEnvironment) Start(ctx context.Context) error { return nil } -func (m *mockEnvironment) Stop(ctx context.Context) error { return nil } -func (m *mockEnvironment) WaitForStop(ctx context.Context, duration time.Duration, terminate bool) error { return nil } +func (m *mockEnvironment) InSituUpdate() error { return nil } +func (m *mockEnvironment) OnBeforeStart(ctx context.Context) error { return nil } +func (m *mockEnvironment) Start(ctx context.Context) error { return nil } +func (m *mockEnvironment) Stop(ctx context.Context) error { return nil } +func (m *mockEnvironment) WaitForStop(ctx context.Context, duration time.Duration, terminate bool) error { + return nil +} func (m *mockEnvironment) Terminate(ctx context.Context, signal string) error { return nil } -func (m *mockEnvironment) Destroy() error { return nil } -func (m *mockEnvironment) ExitState() (uint32, bool, error) { return 0, false, nil } -func (m *mockEnvironment) Create() error { return nil } -func (m *mockEnvironment) Attach(ctx context.Context) error { return nil } -func (m *mockEnvironment) SendCommand(string) error { return nil } -func (m *mockEnvironment) Readlog(int) ([]string, error) { return nil, nil } -func (m *mockEnvironment) State() string { return "offline" } -func (m *mockEnvironment) SetState(string) {} -func (m *mockEnvironment) Uptime(ctx context.Context) (int64, error) { return 0, nil } -func (m *mockEnvironment) SetLogCallback(func([]byte)) {} -func (m *mockEnvironment) SetStream(bool) {} +func (m *mockEnvironment) Destroy() error { return nil } +func (m *mockEnvironment) ExitState() (uint32, bool, error) { return 0, false, nil } +func (m *mockEnvironment) Create() error { return nil } +func (m *mockEnvironment) Attach(ctx context.Context) error { return nil } +func (m *mockEnvironment) SendCommand(string) error { return nil } +func (m *mockEnvironment) Readlog(int) ([]string, error) { return nil, nil } +func (m *mockEnvironment) State() string { return "offline" } +func (m *mockEnvironment) SetState(string) {} +func (m *mockEnvironment) Uptime(ctx context.Context) (int64, error) { return 0, nil } +func (m *mockEnvironment) SetLogCallback(func([]byte)) {} +func (m *mockEnvironment) SetStream(bool) {} // newMockServer creates a minimal Server instance for testing func newMockServer() *Server { @@ -220,7 +222,7 @@ func TestProgressTrackerByteMode(t *testing.T) { defer tracker.Close() // Add bytes written - prog.AddWritten(1024 * 1024) // 1MB + prog.AddWritten(1024 * 1024) // 1MB tracker.CheckProgress() prog.AddWritten(2 * 1024 * 1024) // Another 2MB @@ -284,7 +286,7 @@ func TestProgressEvent(t *testing.T) { // TestS3ProgressSplitLogic tests the 80/20 progress split for S3 func TestS3ProgressSplitLogic(t *testing.T) { // Test S3 progress calculation (80% archive, 20% upload) - + testCases := []struct { archiveBytes int64 totalBytes int64 @@ -320,15 +322,15 @@ func TestProgressTrackerPerformance(t *testing.T) { // Measure time for many rapid progress updates start := time.Now() - + for i := 0; i < 1000; i++ { prog.AddWritten(1000) // 1000 rapid updates tracker.CheckProgress() } - + elapsed := time.Since(start) // Should be very fast (under 100ms for 1000 calls) - assert.True(t, elapsed < 100*time.Millisecond, + assert.True(t, elapsed < 100*time.Millisecond, "1000 CheckProgress calls should be fast, took %v", elapsed) -} \ No newline at end of file +} diff --git a/server/backup_queue_test.go b/server/backup_queue_test.go index 7484659fc..32b37c18e 100644 --- a/server/backup_queue_test.go +++ b/server/backup_queue_test.go @@ -41,17 +41,15 @@ func TestBackupQueueConcurrencyLimits(t *testing.T) { registry := NewBackupOperationRegistry() ctx := context.Background() - var operations []*BackupOperation var cancels []context.CancelFunc // Fill up all 8 backup slots for i := 0; i < 8; i++ { - op, _, cancel, err, wasQueued := registry.Register(ctx, + _, _, cancel, err, wasQueued := registry.Register(ctx, "backup"+string(rune(i+'0')), "server1", OperationTypeBackup) require.NoError(t, err) assert.False(t, wasQueued, "First 8 operations should not be queued") - - operations = append(operations, op) + cancels = append(cancels, cancel) } @@ -71,7 +69,7 @@ func TestBackupQueueConcurrencyLimits(t *testing.T) { // Cleanup: Complete one operation and verify slot becomes available registry.Complete("backup0") - + // Now 9th operation should succeed (it won't queue since slot is available) _, _, cancel9, err, wasQueued := registry.Register(ctx, "backup9", "server1", OperationTypeBackup) require.NoError(t, err) @@ -95,7 +93,7 @@ func TestBackupRestoreSeparateLimits(t *testing.T) { // Fill up all 8 backup slots for i := 0; i < 8; i++ { - _, _, cancel, err, wasQueued := registry.Register(ctx, + _, _, cancel, err, wasQueued := registry.Register(ctx, "backup"+string(rune(i+'0')), "server1", OperationTypeBackup) require.NoError(t, err) assert.False(t, wasQueued) @@ -104,7 +102,7 @@ func TestBackupRestoreSeparateLimits(t *testing.T) { // Fill up all 8 restore slots (should not be affected by backup slots) for i := 0; i < 8; i++ { - _, _, cancel, err, wasQueued := registry.Register(ctx, + _, _, cancel, err, wasQueued := registry.Register(ctx, "restore"+string(rune(i+'0')), "server1", OperationTypeRestore) require.NoError(t, err) assert.False(t, wasQueued, "Restore slots should be independent of backup slots") @@ -115,7 +113,7 @@ func TestBackupRestoreSeparateLimits(t *testing.T) { status := registry.GetQueueStatus() backupStatus := status["backups"].(map[string]any) restoreStatus := status["restores"].(map[string]any) - + assert.Equal(t, 8, backupStatus["active"]) assert.Equal(t, 0, backupStatus["available"]) assert.Equal(t, 8, restoreStatus["active"]) @@ -182,6 +180,11 @@ func TestServerDeletionCleanup(t *testing.T) { _, _, cancel3, err, _ := registry.Register(ctx, "backup3", "server2", OperationTypeBackup) require.NoError(t, err) cancels = append(cancels, cancel3) + defer func() { + for _, c := range cancels { + c() + } + }() // Verify all operations exist assert.Equal(t, 3, registry.Count()) @@ -217,17 +220,17 @@ func TestConcurrentAccess(t *testing.T) { wg.Add(1) go func(goroutineID int) { defer wg.Done() - + for j := 0; j < operationsPerGoroutine; j++ { backupID := "backup_" + string(rune(goroutineID+'0')) + "_" + string(rune(j+'0')) serverID := "server" + string(rune(goroutineID+'0')) - + _, _, cancel, err, _ := registry.Register(ctx, backupID, serverID, OperationTypeBackup) if err != nil { errors <- err return } - + // Immediately complete to free up slots registry.Complete(backupID) if cancel != nil { @@ -247,4 +250,4 @@ func TestConcurrentAccess(t *testing.T) { // Verify registry is clean assert.Equal(t, 0, registry.Count(), "Registry should be empty after all operations completed") -} \ No newline at end of file +} diff --git a/server/backup_restore_test.go b/server/backup_restore_test.go new file mode 100644 index 000000000..c7741a802 --- /dev/null +++ b/server/backup_restore_test.go @@ -0,0 +1,202 @@ +package server + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/server/backup" + "github.com/Rene-Roscher/wings/server/filesystem" +) + +const restoreTestUUID = "6a3c2f6e-1b7d-4c8e-9f0a-2d4b6c8e0f12" + +// restoreTestClient only implements the Panel call that RestoreBackup makes. +type restoreTestClient struct { + remote.Client +} + +func (restoreTestClient) SendRestorationStatus(context.Context, string, bool) error { + return nil +} + +// restoreTestEnvironment reports the container as offline so RestoreBackup does +// not wait for a stop. +type restoreTestEnvironment struct { + environment.ProcessEnvironment +} + +func (restoreTestEnvironment) State() string { return environment.ProcessOfflineState } + +// Fork: RestoreBackupWithContext drives the environment state (restoring -> +// actual state) and checks whether the container is running once it is done, +// so the test environment has to provide these as well. +func (restoreTestEnvironment) SetState(string) {} + +func (restoreTestEnvironment) IsRunning(context.Context) (bool, error) { return false, nil } + +// restoreTestBackup runs a hook before the real local restore, so a test can change +// server state while a restore is in progress. +type restoreTestBackup struct { + *backup.LocalBackup + beforeRestore func() +} + +func (b restoreTestBackup) Restore(ctx context.Context, r io.Reader, cb backup.RestoreCallback) error { + if b.beforeRestore != nil { + b.beforeRestore() + } + return b.LocalBackup.Restore(ctx, r, cb) +} + +// newRestoreTestServer returns a server with a real filesystem rooted in a temp +// directory, and a backup directory. +func newRestoreTestServer(t *testing.T) (*Server, string, string) { + t.Helper() + + backupDir := t.TempDir() + cfg := &config.Configuration{ + AuthenticationToken: "abc", + System: config.SystemConfiguration{ + BackupDirectory: backupDir, + DiskCheckInterval: 150, + }, + } + cfg.System.User.Uid = os.Getuid() + cfg.System.User.Gid = os.Getgid() + config.Set(cfg) + + s, err := New(restoreTestClient{}) + require.NoError(t, err) + s.Config().Uuid = "restore-test" + s.Environment = restoreTestEnvironment{} + + root := filepath.Join(t.TempDir(), "server") + require.NoError(t, os.Mkdir(root, 0o755)) + s.fs, err = filesystem.New(root, 0, nil) + require.NoError(t, err) + + return s, backupDir, root +} + +func writeRestoreTestArchive(t *testing.T, dir string, write func(tw *tar.Writer)) { + t.Helper() + + f, err := os.Create(filepath.Join(dir, restoreTestUUID+".tar.gz")) + require.NoError(t, err) + defer f.Close() + + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + write(tw) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) +} + +func addRestoreTestFile(t *testing.T, tw *tar.Writer, name string, mode int64, body string) { + t.Helper() + + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: name, + Typeflag: tar.TypeReg, + Mode: mode, + Size: int64(len(body)), + ModTime: time.Unix(1700000000, 0), + })) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) +} + +func addRestoreTestHeader(t *testing.T, tw *tar.Writer, hdr *tar.Header) { + t.Helper() + + hdr.ModTime = time.Unix(1700000000, 0) + require.NoError(t, tw.WriteHeader(hdr)) +} + +func restoreTestLocal() backup.BackupInterface { + return backup.NewLocal(restoreTestClient{}, restoreTestUUID, "") +} + +// Entries are restored as the type they have in the archive. +func TestRestoreRecreatesEntryTypes(t *testing.T) { + s, backupDir, root := newRestoreTestServer(t) + writeRestoreTestArchive(t, backupDir, func(tw *tar.Writer) { + addRestoreTestHeader(t, tw, &tar.Header{Name: "data", Typeflag: tar.TypeDir, Mode: 0o755}) + addRestoreTestFile(t, tw, "data/a.txt", 0o644, "hello") + addRestoreTestHeader(t, tw, &tar.Header{Name: "data/link", Typeflag: tar.TypeSymlink, Linkname: "a.txt", Mode: 0o777}) + addRestoreTestHeader(t, tw, &tar.Header{Name: "device", Typeflag: tar.TypeChar, Mode: 0o644, Devmajor: 1, Devminor: 3}) + }) + + require.NoError(t, s.RestoreBackup(restoreTestLocal(), nil, false)) + + st, err := os.Lstat(filepath.Join(root, "data")) + require.NoError(t, err) + assert.True(t, st.IsDir()) + + b, err := os.ReadFile(filepath.Join(root, "data", "a.txt")) + require.NoError(t, err) + assert.Equal(t, "hello", string(b)) + + target, err := os.Readlink(filepath.Join(root, "data", "link")) + require.NoError(t, err) + assert.Equal(t, "a.txt", target) + + _, err = os.Lstat(filepath.Join(root, "device")) + assert.ErrorIs(t, err, os.ErrNotExist, "special files are not restored") +} + +// A restore does not change whether the server is suspended. +func TestRestoreDoesNotChangeSuspension(t *testing.T) { + s, backupDir, _ := newRestoreTestServer(t) + writeRestoreTestArchive(t, backupDir, func(tw *tar.Writer) { + addRestoreTestFile(t, tw, "a.txt", 0o644, "hello") + }) + + // The server is suspended while the restore is running. + b := restoreTestBackup{ + LocalBackup: backup.NewLocal(restoreTestClient{}, restoreTestUUID, ""), + beforeRestore: func() { s.Config().SetSuspended(true) }, + } + require.NoError(t, s.RestoreBackup(b, nil, false)) + assert.True(t, s.IsSuspended()) + + s.Config().SetSuspended(false) + require.NoError(t, s.RestoreBackup(restoreTestLocal(), nil, false)) + assert.False(t, s.IsSuspended()) +} + +// With truncate set, the existing files are deleted before the backup is +// restored, and the server's directory itself is kept. +func TestRestoreTruncatesBeforeRestoring(t *testing.T) { + s, backupDir, root := newRestoreTestServer(t) + writeRestoreTestArchive(t, backupDir, func(tw *tar.Writer) { + addRestoreTestFile(t, tw, "restored.txt", 0o644, "hello") + }) + require.NoError(t, os.MkdirAll(filepath.Join(root, "old", "dir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "old", "dir", "file.txt"), []byte("old"), 0o644)) + before, err := os.Stat(root) + require.NoError(t, err) + + require.NoError(t, s.RestoreBackup(restoreTestLocal(), nil, true)) + + _, err = os.Stat(filepath.Join(root, "old")) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(filepath.Join(root, "restored.txt")) + assert.NoError(t, err) + + after, err := os.Stat(root) + require.NoError(t, err) + assert.True(t, os.SameFile(before, after), "the server directory should be kept") +} diff --git a/server/backup_retry_test.go b/server/backup_retry_test.go index b4a8eaa89..eaf3abc09 100644 --- a/server/backup_retry_test.go +++ b/server/backup_retry_test.go @@ -24,7 +24,7 @@ func TestBackupRetryLogicBasics(t *testing.T) { for _, tc := range testCases { backoff := time.Duration(1<= minExpected, + assert.True(t, elapsed >= minExpected, "Backoff timing should be cumulative, elapsed: %v, expected >= %v", elapsed, minExpected) } @@ -95,7 +95,7 @@ func TestBackupOperationRegistryIntegrationWithRetries(t *testing.T) { _, opCtx, cancel, err, wasQueued := registry.Register(ctx, "retry-test-backup", "server1", OperationTypeBackup) require.NoError(t, err) assert.False(t, wasQueued) - + defer func() { cancel() registry.Complete("retry-test-backup") @@ -154,18 +154,18 @@ func TestFailureEventSimulation(t *testing.T) { // TestBackupRetryStateManagement tests that server states are properly managed during retries func TestBackupRetryStateManagement(t *testing.T) { // Test that atomic state transitions work correctly - + // Simulate initial state isBackingUp := false - + // Start backup (should set backing up) isBackingUp = true assert.True(t, isBackingUp, "Server should be in backing up state") - + // Simulate retry (state should remain backing up) // No state change during retry assert.True(t, isBackingUp, "Server should remain in backing up state during retry") - + // Simulate completion or failure (should clear state) isBackingUp = false assert.False(t, isBackingUp, "Server should clear backing up state after completion/failure") @@ -174,23 +174,23 @@ func TestBackupRetryStateManagement(t *testing.T) { // TestProgressTrackingDuringRetries tests progress is handled correctly during retries func TestProgressTrackingDuringRetries(t *testing.T) { // Test progress reset between retry attempts - + type MockProgress struct { current int64 total int64 } - + progress := &MockProgress{current: 0, total: 100} - + // First attempt - progress to 50% progress.current = 50 assert.Equal(t, int64(50), progress.current) - + // Retry - progress should reset progress.current = 0 assert.Equal(t, int64(0), progress.current, "Progress should reset between retry attempts") - + // Second attempt - progress to completion progress.current = 100 assert.Equal(t, int64(100), progress.current) -} \ No newline at end of file +} diff --git a/server/config_parser.go b/server/config_parser.go index a24436ebe..5387a0b6c 100644 --- a/server/config_parser.go +++ b/server/config_parser.go @@ -22,6 +22,13 @@ func (s *Server) UpdateConfigurationFiles() { f := cf pool.Submit(func() { + // Carry on with the other files if updating this one panics. + defer func() { + if r := recover(); r != nil { + s.Log().WithField("file_name", f.FileName).WithField("panic", r).Error("recovered from panic while updating configuration file") + } + }() + if f.Parser == parser.File { if _, err := s.Filesystem().UnixFS().Stat(f.FileName); errors.Is(err, ufs.ErrNotExist) { s.Log().WithField("file_name", f.FileName).Debug("skipping text configuration file that does not exist yet") @@ -29,16 +36,22 @@ func (s *Server) UpdateConfigurationFiles() { } } - file, err := s.Filesystem().UnixFS().Touch(f.FileName, ufs.O_RDWR|ufs.O_CREATE, 0o644) + // Writes to the file count against the server's disk limit. + file, err := s.Filesystem().Touch(f.FileName, ufs.O_RDWR|ufs.O_CREATE) if err != nil { s.Log().WithField("file_name", f.FileName).WithField("error", err).Error("failed to open file for configuration") return } - defer file.Close() if err := f.Parse(file); err != nil { s.Log().WithField("error", err).Error("failed to parse and update server configuration file") } + if err := file.Close(); err != nil { + s.Log().WithField("file_name", f.FileName).WithField("error", err).Error("failed to close configuration file") + } + if err := s.Filesystem().Chown(f.FileName); err != nil { + s.Log().WithField("file_name", f.FileName).WithField("error", err).Warn("failed to set the owner of configuration file") + } s.Log().WithField("file_name", f.FileName).Debug("finished processing server configuration file") }) diff --git a/server/config_parser_test.go b/server/config_parser_test.go new file mode 100644 index 000000000..100c1a45d --- /dev/null +++ b/server/config_parser_test.go @@ -0,0 +1,74 @@ +package server + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/parser" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/server/filesystem" +) + +func newConfigParserServer(t *testing.T, diskLimit int64, raw string) (*Server, string) { + t.Helper() + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.System.User.Uid = os.Getuid() + cfg.System.User.Gid = os.Getgid() + config.Set(cfg) + + root := filepath.Join(t.TempDir(), "server") + require.NoError(t, os.Mkdir(root, 0o755)) + fs, err := filesystem.New(root, diskLimit, nil) + require.NoError(t, err) + + var cf parser.ConfigurationFile + require.NoError(t, json.Unmarshal([]byte(raw), &cf)) + + s, err := New(nil) + require.NoError(t, err) + s.fs = fs + s.procConfig = &remote.ProcessConfiguration{ConfigurationFiles: []parser.ConfigurationFile{cf}} + return s, root +} + +// Rewriting a configuration file counts against the server's disk limit, so a +// rewrite that would grow the file past it is refused. +func TestConfigurationFilesRespectDiskLimit(t *testing.T) { + raw := `{"file":"log.txt","parser":"file","replace":[{"match":"k","replace_with":"` + strings.Repeat("x", 1024) + `"}]}` + s, root := newConfigParserServer(t, 1<<20, raw) + original := bytes.Repeat([]byte("k\n"), 32768) + require.NoError(t, os.WriteFile(filepath.Join(root, "log.txt"), original, 0o644)) + + _, err := s.Filesystem().DiskUsage(false) + require.NoError(t, err) + + s.UpdateConfigurationFiles() + + // The file is left as it was rather than partly rewritten. + b, err := os.ReadFile(filepath.Join(root, "log.txt")) + require.NoError(t, err) + require.Equal(t, original, b, "the configuration file grew past the disk limit") + require.LessOrEqual(t, s.Filesystem().CachedUsage(), int64(1<<20)) +} + +// Within the disk limit, configuration files are rewritten and counted. +func TestConfigurationFilesAreUpdated(t *testing.T) { + raw := `{"file":"server.properties","parser":"file","replace":[{"match":"port=","replace_with":"port=25565"}]}` + s, root := newConfigParserServer(t, 0, raw) + require.NoError(t, os.WriteFile(filepath.Join(root, "server.properties"), []byte("motd=hi\nport=1\n"), 0o644)) + + s.UpdateConfigurationFiles() + + b, err := os.ReadFile(filepath.Join(root, "server.properties")) + require.NoError(t, err) + require.Equal(t, "motd=hi\nport=25565\n", string(b)) +} diff --git a/server/configuration.go b/server/configuration.go index aaca9f61e..da969f97d 100644 --- a/server/configuration.go +++ b/server/configuration.go @@ -21,9 +21,16 @@ type ConfigurationMeta struct { Description string `json:"description"` } +// Configuration is the configuration of a server, as provided by the Panel. The +// values are kept in an embedded struct so that a new configuration can replace +// them without also overwriting the mutex that guards them. type Configuration struct { mu sync.RWMutex + configurationData +} + +type configurationData struct { // The unique identifier for the server that should be used when referencing // it against the Panel API (and internally). This will be used when naming // docker containers as well as in log output. @@ -67,6 +74,14 @@ func (s *Server) Config() *Configuration { return &s.cfg } +// snapshot returns a copy of the configuration that is read while holding the +// lock, so it is consistent and safe to use without holding the lock. +func (c *Configuration) snapshot() Configuration { + c.mu.RLock() + defer c.mu.RUnlock() + return Configuration{configurationData: c.configurationData} +} + // DiskSpace returns the amount of disk space available to a server in bytes. func (s *Server) DiskSpace() int64 { s.cfg.mu.RLock() diff --git a/server/configuration_test.go b/server/configuration_test.go new file mode 100644 index 000000000..5e035c495 --- /dev/null +++ b/server/configuration_test.go @@ -0,0 +1,103 @@ +package server + +import ( + "encoding/json" + "sync" + "testing" + "time" + + "github.com/Rene-Roscher/wings/remote" +) + +func newConfiguredServer(t *testing.T) *Server { + t.Helper() + setTestConfig(t) + + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + if err := s.SyncWithConfiguration(diskSpaceConfiguration(t, 1)); err != nil { + t.Fatal(err) + } + return s +} + +func diskSpaceConfiguration(t *testing.T, disk int64) remote.ServerConfigurationResponse { + t.Helper() + settings, err := json.Marshal(map[string]any{ + "uuid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + "build": map[string]any{"disk_space": disk}, + }) + if err != nil { + t.Fatal(err) + } + return remote.ServerConfigurationResponse{ + Settings: settings, + ProcessConfiguration: &remote.ProcessConfiguration{}, + } +} + +// Syncing replaces the configuration while other goroutines may be waiting to +// read it, and none of them should be left waiting forever. +func TestSyncWithConfigurationWhileReading(t *testing.T) { + s := newConfiguredServer(t) + responses := []remote.ServerConfigurationResponse{diskSpaceConfiguration(t, 1), diskSpaceConfiguration(t, 2)} + + done := make(chan struct{}) + go func() { + defer close(done) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(2) + go func() { + defer wg.Done() + for j := 0; j < 500; j++ { + if err := s.SyncWithConfiguration(responses[j%2]); err != nil { + t.Error(err) + return + } + } + }() + go func() { + defer wg.Done() + for j := 0; j < 500; j++ { + _ = s.DiskSpace() + _ = s.Config().GetUuid() + _ = s.cfg.snapshot() + } + }() + } + wg.Wait() + }() + + select { + case <-done: + case <-time.After(time.Second * 30): + t.Fatal("timed out waiting for configuration readers and writers to finish") + } + if d := s.DiskSpace(); d != 1024*1024 && d != 2*1024*1024 { + t.Fatalf("expected disk space from one of the synced configurations, got %d", d) + } +} + +// The configuration is returned by the API, so it must keep encoding its +// values at the top level of the object. +func TestConfigurationSnapshotJSON(t *testing.T) { + s := newConfiguredServer(t) + + b, err := json.Marshal(s.cfg.snapshot()) + if err != nil { + t.Fatal(err) + } + var v map[string]json.RawMessage + if err := json.Unmarshal(b, &v); err != nil { + t.Fatal(err) + } + if string(v["uuid"]) != `"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"` { + t.Fatalf("expected uuid at the top level of the configuration, got %s", b) + } + if _, ok := v["build"]; !ok { + t.Fatalf("expected build at the top level of the configuration, got %s", b) + } +} diff --git a/server/console_output_test.go b/server/console_output_test.go new file mode 100644 index 000000000..d081eb431 --- /dev/null +++ b/server/console_output_test.go @@ -0,0 +1,55 @@ +package server + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" +) + +// blockingStateEnvironment holds every call to State until it is released. +type blockingStateEnvironment struct { + environment.ProcessEnvironment + release chan struct{} +} + +func (e *blockingStateEnvironment) State() string { + <-e.release + return environment.ProcessRunningState +} + +// Each line of console output is checked before the next one is read. +func TestConsoleOutputIsCheckedAsItIsRead(t *testing.T) { + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + config.Set(cfg) + + s, err := New(nil) + require.NoError(t, err) + env := &blockingStateEnvironment{release: make(chan struct{})} + s.Environment = env + + done := make(chan struct{}) + go func() { + s.processConsoleOutputEvent([]byte("line")) + close(done) + }() + + select { + case <-done: + t.Fatal("expected the line to be checked before the next one is read") + case <-time.After(100 * time.Millisecond): + } + + close(env.release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("expected the line to be processed once it could be checked") + } +} diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index 30e9d46fe..9b9b235f7 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -9,12 +9,12 @@ import ( "path/filepath" "strings" "sync" + "time" "emperror.dev/errors" "github.com/apex/log" "github.com/juju/ratelimit" "github.com/klauspost/pgzip" - ignore "github.com/sabhiram/go-gitignore" "github.com/Rene-Roscher/wings/config" "github.com/Rene-Roscher/wings/internal/progress" @@ -26,7 +26,7 @@ const memory = 4 * 1024 var pool = sync.Pool{ New: func() interface{} { b := make([]byte, memory) - return b + return &b }, } @@ -85,7 +85,7 @@ type Archive struct { func (a *Archive) Create(ctx context.Context, dst string) error { // Using os.OpenFile here is expected, as long as `dst` is not a user // provided path. - f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) //nolint:gosec // dst is never user provided, see above if err != nil { return err } @@ -129,6 +129,36 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { a.Files = files } + // If we're specifically looking for only certain files, or have requested + // that certain files be ignored we'll update the callback function to reflect + // that request. Ignore patterns are compiled before anything is written so that + // an invalid list does not leave an empty archive behind. + var callback walkFunc + if len(a.Files) == 0 && len(a.Ignore) > 0 { + i, err := compileIgnore(a.Ignore) + if err != nil { + return errors.WrapIf(err, "filesystem: unable to compile archive ignore patterns") + } + // Time each evaluation and abort if the patterns prove disproportionately + // expensive for this server's file tree. + budget := newIgnoreMatchBudget(ignorePatternCount(a.Ignore)) + callback = a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error { + start := time.Now() + skip := i.MatchesPath(relative) + if err := budget.track(time.Since(start)); err != nil { + return err + } + if skip { + return SkipThis + } + return nil + }) + } else if len(a.Files) > 0 { + callback = a.withFilesCallback() + } else { + callback = a.callback() + } + // Create compressor based on configured format compressor, err := a.createCompressor(w) if err != nil { @@ -152,24 +182,6 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { fs := a.Filesystem.unixFS - // If we're specifically looking for only certain files, or have requested - // that certain files be ignored we'll update the callback function to reflect - // that request. - var callback walkFunc - if len(a.Files) == 0 && len(a.Ignore) > 0 { - i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...) - callback = a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error { - if i.MatchesPath(relative) { - return SkipThis - } - return nil - }) - } else if len(a.Files) > 0 { - callback = a.withFilesCallback() - } else { - callback = a.callback() - } - // Open the base directory we were provided. dirfd, name, closeFd, err := fs.SafePath(a.BaseDirectory) defer closeFd() @@ -279,10 +291,10 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn // the logs, but we're not going to stop the backup. There are far too many cases of // symlinks causing all sorts of unnecessary pain in this process. Sucks to suck if // it doesn't work. - target, err = os.Readlink(s.Name()) + target, err = a.Filesystem.unixFS.Readlinkat(dirfd, name) if err != nil { // Ignore the not exist errors specifically, since there is nothing important about that. - if !os.IsNotExist(err) { + if !errors.Is(err, ufs.ErrNotExist) { log.WithField("name", name).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...") } return nil @@ -294,11 +306,7 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn if err != nil { return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", name) } - - // Fix the header name if the file is not a symlink. - if s.Mode()&fs.ModeSymlink == 0 { - header.Name = relative - } + header.Name = relative // Write the tar FileInfoHeader to the archive. if err := a.w.WriteHeader(header); err != nil { @@ -317,11 +325,9 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn buf = make([]byte, header.Size) } else { // Get a fixed-size buffer from the pool to save on allocations. - buf = pool.Get().([]byte) - defer func() { - buf = make([]byte, memory) - pool.Put(buf) - }() + bp := pool.Get().(*[]byte) + defer pool.Put(bp) + buf = *bp } // Open the file. diff --git a/server/filesystem/archive_links_test.go b/server/filesystem/archive_links_test.go new file mode 100644 index 000000000..4500dad5c --- /dev/null +++ b/server/filesystem/archive_links_test.go @@ -0,0 +1,94 @@ +package filesystem + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newArchiveTestFilesystem returns a filesystem rooted in a temporary directory. +func newArchiveTestFilesystem(t *testing.T) *Filesystem { + t.Helper() + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + return fs +} + +// archiveEntries streams a full archive of the filesystem and returns its +// tar headers keyed by entry name. +func archiveEntries(t *testing.T, fsys *Filesystem) map[string]*tar.Header { + t.Helper() + + var buf bytes.Buffer + a := &Archive{Filesystem: fsys} + require.NoError(t, a.Stream(context.Background(), &buf)) + + gz, err := gzip.NewReader(&buf) + require.NoError(t, err) + tr := tar.NewReader(gz) + + entries := make(map[string]*tar.Header) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + entries[hdr.Name] = hdr + } + return entries +} + +func archiveEntryNames(entries map[string]*tar.Header) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// A symlink in a subdirectory of the server is archived under its own path, with +// its own target. +func TestArchiveKeepsSymlinkEntriesWithTheirPath(t *testing.T) { + fsys := newArchiveTestFilesystem(t) + root := fsys.Path() + require.NoError(t, os.Mkdir(filepath.Join(root, "sub"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "sub", "target.txt"), []byte("data"), 0o644)) + require.NoError(t, os.Symlink("target.txt", filepath.Join(root, "sub", "link"))) + + entries := archiveEntries(t, fsys) + + entry, ok := entries["sub/link"] + require.True(t, ok, "symlink sub/link is missing from the archive; entries: %v", archiveEntryNames(entries)) + assert.Equal(t, byte(tar.TypeSymlink), entry.Typeflag) + assert.Equal(t, "target.txt", entry.Linkname) +} + +// The link target is read from the link inside the server. +func TestArchiveReadsSymlinkTargetFromServer(t *testing.T) { + fsys := newArchiveTestFilesystem(t) + root := fsys.Path() + require.NoError(t, os.WriteFile(filepath.Join(root, "target.txt"), []byte("data"), 0o644)) + require.NoError(t, os.Symlink("target.txt", filepath.Join(root, "link"))) + + // A link with the same name in the working directory. + cwd := t.TempDir() + require.NoError(t, os.Symlink("other.txt", filepath.Join(cwd, "link"))) + t.Chdir(cwd) + + entries := archiveEntries(t, fsys) + + entry, ok := entries["link"] + require.True(t, ok, "symlink link is missing from the archive; entries: %v", archiveEntryNames(entries)) + assert.Equal(t, "target.txt", entry.Linkname) +} diff --git a/server/filesystem/archive_restore.go b/server/filesystem/archive_restore.go index 89cb5e436..f88dd870d 100644 --- a/server/filesystem/archive_restore.go +++ b/server/filesystem/archive_restore.go @@ -23,14 +23,14 @@ func DetectCompressionFormat(reader io.ReadCloser) (CompressionFormat, io.ReadCl if reader == nil { return CompressionGzip, nil, errors.New("backup: nil reader provided to format detection") } - + // Peek first 4 bytes without consuming the stream peekReader := bufio.NewReader(reader) header, err := peekReader.Peek(4) if err != nil && err != io.EOF { return CompressionGzip, io.NopCloser(peekReader), errors.Wrap(err, "backup: failed to read format detection header") } - + // Validate we have enough data for detection if len(header) < 2 { return CompressionGzip, io.NopCloser(peekReader), errors.New("backup: insufficient data for format detection") @@ -51,7 +51,7 @@ func CreateDecompressor(reader io.ReadCloser, format CompressionFormat) (io.Read if reader == nil { return nil, errors.New("backup: nil reader provided to decompressor") } - + switch format { case CompressionGzip: gzReader, err := gzip.NewReader(reader) @@ -74,5 +74,3 @@ func CreateDecompressor(reader io.ReadCloser, format CompressionFormat) (io.Read return gzReader, nil } } - - diff --git a/server/filesystem/archive_stream_test.go b/server/filesystem/archive_stream_test.go index 26e3fe964..3fd1a31e4 100644 --- a/server/filesystem/archive_stream_test.go +++ b/server/filesystem/archive_stream_test.go @@ -84,6 +84,45 @@ func TestArchive_Stream(t *testing.T) { g.Assert(files).Equal(expected) }) + + g.It("treats regular expression metacharacters in ignore patterns literally", func() { + g.Assert(fs.CreateDirectory("world (copy)", "/")).IsNil() + g.Assert(fs.CreateDirectory("logs", "/")).IsNil() + for _, name := range []string{"world (copy)/level.dat", "logs/latest.log", "server.jar"} { + r := strings.NewReader("hello, world!\n") + g.Assert(fs.Write(name, r, r.Size(), 0o644)).IsNil() + } + + a := &Archive{ + Filesystem: fs, + Ignore: "world (copy)/\n*.log\n", + } + + archivePath := filepath.Join(rfs.root, "archive.tar.gz") + g.Assert(a.Create(context.Background(), archivePath)).IsNil() + + genericFs, err := archives.FileSystem(context.Background(), archivePath, nil) + g.Assert(err).IsNil() + afs, ok := genericFs.(iofs.ReadDirFS) + g.Assert(ok).IsTrue() + + files, err := getFiles(afs, ".") + g.Assert(err).IsNil() + g.Assert(files).Equal([]string{"server.jar"}) + }) + + g.It("rejects an ignore list that exceeds the limits", func() { + r := strings.NewReader("hello, world!\n") + g.Assert(fs.Write("server.jar", r, r.Size(), 0o644)).IsNil() + + a := &Archive{ + Filesystem: fs, + Ignore: strings.Repeat("*a", MaxIgnorePatternWildcards+1), + } + + archivePath := filepath.Join(rfs.root, "archive.tar.gz") + g.Assert(a.Create(context.Background(), archivePath)).IsNotNil() + }) }) } @@ -107,8 +146,10 @@ func getFiles(f iofs.ReadDirFS, name string) ([]string, error) { return nil, err } + // The fork's archiver preserves empty directories as their own + // entries; skip them here since only files are listed. if files == nil { - return nil, nil + continue } v = append(v, files...) diff --git a/server/filesystem/archiverext/compressed.go b/server/filesystem/archiverext/compressed.go deleted file mode 100644 index 536a25f70..000000000 --- a/server/filesystem/archiverext/compressed.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2016 Matthew Holt - -// Code in this file was derived from -// https://github.com/mholt/archiver/blob/v4.0.0-alpha.8/fs.go -// -// These modifications were necessary to allow us to use an already open file -// with archiver.FileFS. - -package archiverext - -import ( - "io" - "io/fs" - - "github.com/mholt/archives" -) - -// FileFS allows accessing a file on disk using a consistent file system interface. -// The value should be the path to a regular file, not a directory. This file will -// be the only entry in the file system and will be at its root. It can be accessed -// within the file system by the name of "." or the filename. -// -// If the file is compressed, set the Compression field so that reads from the -// file will be transparently decompressed. -type FileFS struct { - // File is the compressed file backing the FileFS. - File fs.File - - // If file is compressed, setting this field will - // transparently decompress reads. - Compression archives.Decompressor -} - -// Open opens the named file, which must be the file used to create the file system. -func (f FileFS) Open(name string) (fs.File, error) { - if err := f.checkName(name, "open"); err != nil { - return nil, err - } - if f.Compression == nil { - return f.File, nil - } - r, err := f.Compression.OpenReader(f.File) - if err != nil { - return nil, err - } - return compressedFile{f.File, r}, nil -} - -// ReadDir returns a directory listing with the file as the singular entry. -func (f FileFS) ReadDir(name string) ([]fs.DirEntry, error) { - if err := f.checkName(name, "stat"); err != nil { - return nil, err - } - info, err := f.Stat(name) - if err != nil { - return nil, err - } - return []fs.DirEntry{fs.FileInfoToDirEntry(info)}, nil -} - -// Stat stats the named file, which must be the file used to create the file system. -func (f FileFS) Stat(name string) (fs.FileInfo, error) { - if err := f.checkName(name, "stat"); err != nil { - return nil, err - } - return f.File.Stat() -} - -func (f FileFS) checkName(name, op string) error { - if !fs.ValidPath(name) { - return &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} - } - // TODO: we may need better name validation. - if name != "." { - return &fs.PathError{Op: op, Path: name, Err: fs.ErrNotExist} - } - return nil -} - -// compressedFile is an fs.File that specially reads -// from a decompression reader, and which closes both -// that reader and the underlying file. -type compressedFile struct { - fs.File - decomp io.ReadCloser -} - -func (cf compressedFile) Read(p []byte) (int, error) { - return cf.decomp.Read(p) -} - -func (cf compressedFile) Close() error { - err := cf.File.Close() - err2 := cf.decomp.Close() - if err2 != nil && err == nil { - err = err2 - } - return err -} diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 485606eb2..5cb065c26 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -1,6 +1,7 @@ package filesystem import ( + "archive/tar" "context" "fmt" "io" @@ -9,7 +10,6 @@ import ( "path" "path/filepath" "strings" - "sync/atomic" "time" "emperror.dev/errors" @@ -17,7 +17,6 @@ import ( "github.com/mholt/archives" "github.com/Rene-Roscher/wings/internal/ufs" - "github.com/Rene-Roscher/wings/server/filesystem/archiverext" ) // CompressFiles compresses all the files matching the given paths in the @@ -31,77 +30,58 @@ import ( // `archive-{date}.tar.gz`. func (fs *Filesystem) CompressFiles(dir string, paths []string) (ufs.FileInfo, error) { a := &Archive{Filesystem: fs, BaseDirectory: dir, Files: paths} - d := path.Join( - dir, - fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")), + name := fmt.Sprintf("archive-%s", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")) + + // Never write into an existing file, such as an archive created by another + // request within the same second. + var ( + d string + f ufs.File + err error ) - f, err := fs.unixFS.OpenFile(d, ufs.O_WRONLY|ufs.O_CREATE, 0o644) - if err != nil { - return nil, err - } - defer f.Close() - cw := ufs.NewCountedWriter(f) - if err := a.Stream(context.Background(), cw); err != nil { - return nil, err - } - if cw.BytesWritten() < 0 || !fs.unixFS.CanFit(cw.BytesWritten()) { - _ = fs.unixFS.Remove(d) - return nil, newFilesystemError(ErrCodeDiskSpace, nil) - } - fs.unixFS.Add(cw.BytesWritten()) - return f.Stat() -} - -func (fs *Filesystem) archiverFileSystem(ctx context.Context, p string) (iofs.FS, io.Closer, error) { - f, err := fs.unixFS.Open(p) - if err != nil { - return nil, nil, err - } - // Do not use defer to close `f`, it will likely be used later. - - format, _, err := archives.Identify(ctx, filepath.Base(p), f) - if err != nil && !errors.Is(err, archives.NoMatch) { - _ = f.Close() - return nil, nil, err + for i := 0; ; i++ { + d = path.Join(dir, name+".tar.gz") + if i > 0 { + d = path.Join(dir, fmt.Sprintf("%s-%d.tar.gz", name, i)) + } + f, err = fs.unixFS.OpenFile(d, ufs.O_WRONLY|ufs.O_CREATE|ufs.O_EXCL, 0o644) + if err == nil { + break + } + if !errors.Is(err, ufs.ErrExist) || i >= 100 { + return nil, err + } } - // Reset the file reader. - if _, err := f.Seek(0, io.SeekStart); err != nil { - _ = f.Close() - return nil, nil, err + // Count every write to the archive against the disk limit as it happens, so + // that archives being created at the same time cannot each use all the space + // that is left. + qf := newQuotaFile(fs, f, 0) + if err := a.Stream(context.Background(), qf); err != nil { + _ = qf.Close() + _ = fs.unixFS.Remove(d) + return nil, err } - - info, err := f.Stat() + st, err := qf.Stat() if err != nil { - _ = f.Close() - return nil, nil, err + _ = qf.Close() + return nil, err } - - if format != nil { - switch ff := format.(type) { - case archives.Zip: - // zip.Reader is more performant than ArchiveFS, because zip.Reader caches content information - // and zip.Reader can open several content files concurrently because of io.ReaderAt requirement - // while ArchiveFS can't. - // zip.Reader doesn't suffer from issue #330 and #310 according to local test (but they should be fixed anyway) - reader, err := zip.NewReader(f, info.Size()) - if err != nil { - _ = f.Close() - return nil, nil, err - } - return reader, f, nil - case archives.Extraction: - return &archives.ArchiveFS{Stream: io.NewSectionReader(f, 0, info.Size()), Format: ff, Context: ctx}, f, nil - case archives.Compression: - return archiverext.FileFS{File: f, Compression: ff}, f, nil - } + if err := qf.Close(); err != nil { + return nil, err } - _ = f.Close() - return nil, nil, archives.NoMatch + return st, nil } +// maxArchiveEntryNameLength is the longest name an entry in an archive can have, +// which matches the longest path that can be created on Linux. +const maxArchiveEntryNameLength = 4096 + // SpaceAvailableForDecompression looks through a given archive and determines // if decompressing it would put the server over its allocated disk space limit. +// +// The entries are read one at a time rather than opening the archive as a file +// system, which is much faster for large archives. func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir string, file string) error { // Don't waste time trying to determine this if we know the server will have the space for // it since there is no limit. @@ -109,46 +89,74 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st return nil } - fsys, archive, err := fs.archiverFileSystem(ctx, filepath.Join(dir, file)) + f, err := fs.unixFS.Open(filepath.Join(dir, file)) if err != nil { - if errors.Is(err, archives.NoMatch) { - return newFilesystemError(ErrCodeUnknownArchive, err) - } return err } - defer archive.Close() + defer f.Close() - var size atomic.Int64 - return iofs.WalkDir(fsys, ".", func(path string, d iofs.DirEntry, err error) error { - if err != nil { - return err - } + format, _, err := archives.Identify(ctx, filepath.Base(file), f) + if err != nil && !errors.Is(err, archives.NoMatch) { + return err + } + // Reset the file reader. + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + info, err := f.Stat() + if err != nil { + return err + } + var size int64 + add := func(name string, n int64) error { select { case <-ctx.Done(): - // Stop walking if the context is canceled. return ctx.Err() default: - info, err := d.Info() - if err != nil { - return err - } - fileSize := info.Size() - if fileSize <= 0 { - return nil - } - current := size.Load() - if fileSize > math.MaxInt64-current { + } + if len(name) > maxArchiveEntryNameLength { + return errors.New("filesystem: archive contains an entry with a name that is too long") + } + if n <= 0 { + return nil + } + if n > math.MaxInt64-size { + return newFilesystemError(ErrCodeDiskSpace, nil) + } + size += n + if !fs.unixFS.CanFit(size) { + return newFilesystemError(ErrCodeDiskSpace, nil) + } + return nil + } + + switch ff := format.(type) { + case archives.Zip: + // The sizes can be read straight from the central directory of a zip. + reader, err := zip.NewReader(f, info.Size()) + if err != nil { + return err + } + for _, e := range reader.File { + if e.UncompressedSize64 > math.MaxInt64 { return newFilesystemError(ErrCodeDiskSpace, nil) } - next := current + fileSize - if !fs.unixFS.CanFit(next) { - return newFilesystemError(ErrCodeDiskSpace, nil) + if err := add(e.Name, int64(e.UncompressedSize64)); err != nil { + return err } - size.Store(next) - return nil } - }) + return nil + case archives.Extraction: + return ff.Extract(ctx, io.NewSectionReader(f, 0, info.Size()), func(ctx context.Context, e archives.FileInfo) error { + return add(e.NameInArchive, e.Size()) + }) + case archives.Compression: + // A single compressed file is checked against the disk limit as it is + // decompressed, since its size is not known until then. + return add(file, info.Size()) + } + return newFilesystemError(ErrCodeUnknownArchive, archives.NoMatch) } // DecompressFile will decompress a file in a given directory by using the @@ -163,6 +171,7 @@ func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file strin } defer f.Close() + // Identify the type of archive we are dealing with. format, input, err := archives.Identify(ctx, filepath.Base(file), f) if err != nil { if errors.Is(err, archives.NoMatch) { @@ -179,8 +188,10 @@ func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file strin }) } -// ExtractStreamUnsafe . -func (fs *Filesystem) ExtractStreamUnsafe(ctx context.Context, dir string, r io.Reader) error { +// ExtractTransfer extracts the archive of a server that is being transferred to +// this node into the server's directory. Every file is kept, including ones the +// denylist would refuse, since they are the server's own files. +func (fs *Filesystem) ExtractTransfer(ctx context.Context, r io.Reader) error { format, input, err := archives.Identify(ctx, "archive.tar.gz", r) if err != nil { if errors.Is(err, archives.NoMatch) { @@ -189,9 +200,10 @@ func (fs *Filesystem) ExtractStreamUnsafe(ctx context.Context, dir string, r io. return err } return fs.extractStream(ctx, extractStreamOptions{ - Directory: dir, - Format: format, - Reader: input, + Directory: "/", + Format: format, + Reader: input, + KeepDenylist: true, }) } @@ -204,6 +216,8 @@ type extractStreamOptions struct { Format archives.Format // Reader for the archive. Reader io.Reader + // KeepDenylist extracts files the denylist would otherwise skip. + KeepDenylist bool } func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptions) error { @@ -231,44 +245,11 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptio } defer reader.Close() - // Open the file for creation/writing - f, err := fs.unixFS.OpenFile(p, ufs.O_WRONLY|ufs.O_CREATE, 0o644) - if err != nil { - return err - } - defer f.Close() - - // Read in 4 KB chunks - buf := make([]byte, 4096) - for { - n, err := reader.Read(buf) - if n > 0 { - - // Check quota before writing the chunk - if quotaErr := fs.HasSpaceFor(int64(n)); quotaErr != nil { - return quotaErr - } - - // Write the chunk - if _, writeErr := f.Write(buf[:n]); writeErr != nil { - return writeErr - } - - // Add to quota - fs.addDisk(int64(n)) - } - - if err != nil { - // EOF are expected - if err == io.EOF { - break - } - - // Return any other - return err - } + // Write the file, counting each write against the disk limit. The file is + // removed if it does not fit. + if _, err := fs.WriteFrom(p, reader); err != nil { + return wrapError(err, opts.FileName) } - return nil } @@ -276,8 +257,10 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptio return ex.Extract(ctx, opts.Reader, func(ctx context.Context, f archives.FileInfo) error { p := filepath.Join(opts.Directory, f.NameInArchive) // If it is ignored, just don't do anything with the entry and skip over it. - if err := fs.IsIgnored(p); err != nil { - return nil + if !opts.KeepDenylist { + if err := fs.IsIgnored(p); err != nil { + return nil + } } // Create directories explicitly; an empty one has no file to create it // implicitly and would otherwise be dropped during extraction. @@ -287,12 +270,30 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptio } return nil } + if f.Mode()&iofs.ModeSymlink != 0 { + if f.LinkTarget == "" { + return nil + } + if err := fs.mkdirAll(filepath.Dir(p), 0o755); err != nil { + return wrapError(err, opts.FileName) + } + if err := fs.Symlink(f.LinkTarget, p); err != nil && !errors.Is(err, ufs.ErrExist) { + return wrapError(err, opts.FileName) + } + return nil + } + // Hard links, devices and other special files are not extracted. + if !f.Mode().IsRegular() { + return nil + } + if h, ok := f.Header.(*tar.Header); ok && h.Typeflag == tar.TypeLink { + return nil + } r, err := f.Open() if err != nil { return err } defer r.Close() - if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { return wrapError(err, opts.FileName) } @@ -303,5 +304,3 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptio return nil }) } - - diff --git a/server/filesystem/compress_binary_test.go b/server/filesystem/compress_binary_test.go index 7fd818775..1566b8041 100644 --- a/server/filesystem/compress_binary_test.go +++ b/server/filesystem/compress_binary_test.go @@ -19,7 +19,7 @@ import ( func TestBinaryCompressionIntegrity(t *testing.T) { // Use the test filesystem helper fs, _ := NewFs() - + tmpDir := fs.Path() // Use the filesystem's root directory // Test with different binary files @@ -55,7 +55,7 @@ func TestBinaryCompressionIntegrity(t *testing.T) { // Copy the binary to our test directory binaryName := filepath.Base(tc.binaryPath) testBinaryPath := filepath.Join(tmpDir, binaryName) - + err := copyFile(tc.binaryPath, testBinaryPath) require.NoError(t, err) @@ -80,7 +80,7 @@ func TestBinaryCompressionIntegrity(t *testing.T) { archiveInfo, err := fs.CompressFiles("", []string{binaryName}) require.NoError(t, err) require.NotNil(t, archiveInfo) - + archivePath := filepath.Join(tmpDir, archiveInfo.Name()) t.Logf("Created archive: %s (size: %d bytes)", archivePath, archiveInfo.Size()) @@ -99,11 +99,11 @@ func TestBinaryCompressionIntegrity(t *testing.T) { // Verify binary was restored restoredInfo, err := os.Stat(testBinaryPath) require.NoError(t, err, "Binary should be restored") - + // Check permissions restoredMode := restoredInfo.Mode() t.Logf("Restored binary permissions: %v", restoredMode) - + // Check if executable bit is preserved (at least for owner) assert.True(t, restoredMode&0100 != 0, "Binary should have executable permission") @@ -121,19 +121,19 @@ func TestBinaryCompressionIntegrity(t *testing.T) { if err != nil { t.Logf("Error executing restored binary: %v", err) t.Logf("Output: %s", string(restoredOutput)) - + // Try to get more info about the failure if exitErr, ok := err.(*exec.ExitError); ok { t.Logf("Exit code: %d", exitErr.ExitCode()) } - + // Check with ldd if it's a library issue lddCmd := exec.Command("ldd", testBinaryPath) lddOutput, _ := lddCmd.CombinedOutput() t.Logf("ldd output:\n%s", string(lddOutput)) } require.NoError(t, err, "Restored binary should execute successfully") - + // Verify output is the same assert.Equal(t, string(output), string(restoredOutput), "Binary output should be identical") @@ -207,17 +207,17 @@ func TestCompressionWithMultipleBinaries(t *testing.T) { // Copy multiple binaries binaries := []string{"/bin/ls", "/bin/cat", "/bin/echo"} checksums := make(map[string]string) - + for _, bin := range binaries { if _, err := os.Stat(bin); os.IsNotExist(err) { continue } - + name := filepath.Base(bin) dst := filepath.Join(tmpDir, name) err := copyFile(bin, dst) require.NoError(t, err) - + checksum, err := calculateChecksum(dst) require.NoError(t, err) checksums[name] = checksum @@ -232,7 +232,7 @@ func TestCompressionWithMultipleBinaries(t *testing.T) { for name := range checksums { files = append(files, name) } - + archiveInfo, err := fs.CompressFiles("", files) require.NoError(t, err) @@ -250,9 +250,9 @@ func TestCompressionWithMultipleBinaries(t *testing.T) { for name, originalChecksum := range checksums { restoredChecksum, err := calculateChecksum(filepath.Join(tmpDir, name)) require.NoError(t, err) - assert.Equal(t, originalChecksum, restoredChecksum, + assert.Equal(t, originalChecksum, restoredChecksum, "Binary %s should be identical after restore", name) - + // Test execution cmd := exec.Command(filepath.Join(tmpDir, name), "--version") _, err = cmd.CombinedOutput() @@ -262,7 +262,7 @@ func TestCompressionWithMultipleBinaries(t *testing.T) { if exitErr, ok := err.(*exec.ExitError); ok { // Exit code 1 or 2 is usually "bad argument", which is fine // Exit code > 128 usually means signal (segfault, etc), which is bad - assert.Less(t, exitErr.ExitCode(), 128, + assert.Less(t, exitErr.ExitCode(), 128, "Binary %s crashed with signal", name) } } @@ -332,32 +332,32 @@ func TestCompressionConsistency(t *testing.T) { // Run multiple compress/decompress cycles for i := 0; i < 5; i++ { t.Logf("Cycle %d", i+1) - + // Compress archiveInfo, err := fs.CompressFiles("", []string{binaryName}) require.NoError(t, err) - + // Delete err = os.Remove(testPath) require.NoError(t, err) - + // Decompress err = fs.DecompressFile(context.Background(), "", archiveInfo.Name()) require.NoError(t, err) - + // Verify checksum checksum, err := calculateChecksum(testPath) require.NoError(t, err) - assert.Equal(t, originalChecksum, checksum, + assert.Equal(t, originalChecksum, checksum, "Checksum should remain consistent after cycle %d", i+1) - + // Test execution cmd := exec.Command(testPath, "test", fmt.Sprintf("cycle-%d", i+1)) output, err := cmd.CombinedOutput() require.NoError(t, err, "Binary should execute after cycle %d", i+1) assert.Contains(t, string(output), fmt.Sprintf("cycle-%d", i+1)) - + // Clean up archive for next cycle os.Remove(filepath.Join(tmpDir, archiveInfo.Name())) } -} \ No newline at end of file +} diff --git a/server/filesystem/compress_test.go b/server/filesystem/compress_test.go index 202cc678d..cde1708cd 100644 --- a/server/filesystem/compress_test.go +++ b/server/filesystem/compress_test.go @@ -7,7 +7,10 @@ import ( "compress/gzip" "context" "os" + "strconv" + "strings" "testing" + "time" . "github.com/franela/goblin" ) @@ -98,6 +101,118 @@ func TestFilesystem_DecompressFileEmptyDirectory(t *testing.T) { }) } +// deepArchive builds a zip or tar.gz archive holding the given number of files, +// all nested in the same deep directory. +func deepArchive(format string, entries int, depth int) ([]byte, error) { + var buf bytes.Buffer + prefix := strings.Repeat("a/", depth) + content := []byte("hello") + if format == "zip" { + zw := zip.NewWriter(&buf) + for i := 0; i < entries; i++ { + w, err := zw.Create(prefix + "f" + strconv.Itoa(i) + ".txt") + if err != nil { + return nil, err + } + if _, err := w.Write(content); err != nil { + return nil, err + } + } + if err := zw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil + } + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for i := 0; i < entries; i++ { + h := &tar.Header{Name: prefix + "f" + strconv.Itoa(i) + ".txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(content)), Format: tar.FormatPAX} + if err := tw.WriteHeader(h); err != nil { + return nil, err + } + if _, err := tw.Write(content); err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Checking the size of an archive takes time in proportion to the archive, however +// deeply nested its entries are. +func TestFilesystem_SpaceAvailableForDecompressionDeepArchive(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + + g.Describe("SpaceAvailableForDecompression", func() { + for _, ext := range []string{"zip", "rar", "tar", "tar.gz"} { + g.It("reads the files in a "+ext, func() { + c, err := os.ReadFile("./testdata/test." + ext) + g.Assert(err).IsNil() + g.Assert(rfs.CreateServerFile("./test."+ext, c)).IsNil() + + fs.SetDiskLimit(1024 * 1024 * 1024) + g.Assert(fs.SpaceAvailableForDecompression(context.Background(), "/", "test."+ext)).IsNil() + }) + } + + for _, format := range []string{"zip", "tar.gz"} { + g.It("rejects a "+format+" holding more than the space available", func() { + // Ten files of five bytes each. + b, err := deepArchive(format, 10, 1) + g.Assert(err).IsNil() + g.Assert(rfs.CreateServerFile("full."+format, b)).IsNil() + + usage, err := fs.DiskUsage(false) + g.Assert(err).IsNil() + fs.SetDiskLimit(usage + 49) + err = fs.SpaceAvailableForDecompression(context.Background(), "/", "full."+format) + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() + + fs.SetDiskLimit(usage + 50) + g.Assert(fs.SpaceAvailableForDecompression(context.Background(), "/", "full."+format)).IsNil() + }) + } + + for _, format := range []string{"zip", "tar.gz"} { + g.It("checks deeply nested entries in a "+format+" in linear time", func() { + g.Timeout(time.Minute) + fs.SetDiskLimit(1024 * 1024 * 1024) + + b, err := deepArchive(format, 400, 2000) + g.Assert(err).IsNil() + g.Assert(rfs.CreateServerFile("deep."+format, b)).IsNil() + + start := time.Now() + g.Assert(fs.SpaceAvailableForDecompression(context.Background(), "/", "deep."+format)).IsNil() + if d := time.Since(start); d > time.Second { + g.Failf("checking the archive took %s", d) + } + }) + + g.It("rejects entries in a "+format+" with names that are too long", func() { + fs.SetDiskLimit(1024 * 1024 * 1024) + + b, err := deepArchive(format, 1, 4000) + g.Assert(err).IsNil() + g.Assert(rfs.CreateServerFile("long."+format, b)).IsNil() + + g.Assert(fs.SpaceAvailableForDecompression(context.Background(), "/", "long."+format) == nil).IsFalse() + }) + } + + g.AfterEach(func() { + fs.SetDiskLimit(0) + _ = fs.TruncateRootDirectory() + }) + }) +} + // zipWithEmptyDir builds a zip holding one file and an empty directory ("empty/"). func zipWithEmptyDir() ([]byte, error) { var buf bytes.Buffer diff --git a/server/filesystem/decompress_test.go b/server/filesystem/decompress_test.go new file mode 100644 index 000000000..415ec9b15 --- /dev/null +++ b/server/filesystem/decompress_test.go @@ -0,0 +1,148 @@ +package filesystem + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + iofs "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" +) + +func gzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write(data) + require.NoError(t, err) + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// A single compressed file decompressed over an existing, larger file leaves +// exactly the decompressed content behind. +func TestDecompressSingleFileReplacesExistingContent(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + require.NoError(t, rfs.CreateServerFileFromString("in.txt", strings.Repeat("A", 100))) + require.NoError(t, rfs.CreateServerFile("in.txt.gz", gzipBytes(t, []byte("hello")))) + + require.NoError(t, fs.DecompressFile(context.Background(), "/", "in.txt.gz")) + + got, err := os.ReadFile(filepath.Join(rfs.root, "server", "in.txt")) + require.NoError(t, err) + assert.Equal(t, "hello", string(got)) +} + +// A single file decompressed from a compressed archive is owned by the server +// user, like every other extracted file. +func TestDecompressSingleFileOwnedByServerUser(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("giving files to another user needs root") + } + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const uid, gid = 1234, 1234 + cfg := config.Get() + cfg.System.User.Uid = uid + cfg.System.User.Gid = gid + config.Set(cfg) + fs.isTest = false + + require.NoError(t, rfs.CreateServerFile("in.txt.gz", gzipBytes(t, []byte("hello")))) + require.NoError(t, fs.DecompressFile(context.Background(), "/", "in.txt.gz")) + + info, err := os.Stat(filepath.Join(rfs.root, "server", "in.txt")) + require.NoError(t, err) + st, ok := info.Sys().(*syscall.Stat_t) + require.True(t, ok) + assert.Equal(t, uint32(uid), st.Uid) + assert.Equal(t, uint32(gid), st.Gid) +} + +// A single compressed file that does not fit the disk limit is not left +// partly written. +func TestDecompressSingleFileStopsAtDiskLimit(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const limit = 1 << 20 + fs.SetDiskLimit(limit) + require.NoError(t, rfs.CreateServerFile("large.gz", gzipBytes(t, make([]byte, 8<<20)))) + + err := fs.DecompressFile(context.Background(), "/", "large.gz") + require.Error(t, err) + assert.True(t, IsErrorCode(err, ErrCodeDiskSpace), "unexpected error: %v", err) + + _, err = os.Stat(filepath.Join(rfs.root, "server", "large")) + assert.ErrorIs(t, err, os.ErrNotExist) + assert.LessOrEqual(t, fs.CachedUsage(), int64(limit)) +} + +// Symlinks in an archive are extracted as symlinks, and special files are not +// extracted. +func TestDecompressExtractsEntriesByType(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "file.txt", Mode: 0o644, Size: 5, Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "link", Linkname: "file.txt", Mode: 0o777, Typeflag: tar.TypeSymlink})) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "hard", Linkname: "file.txt", Mode: 0o644, Typeflag: tar.TypeLink})) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "device", Mode: 0o644, Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3})) + require.NoError(t, tw.WriteHeader(&tar.Header{Name: "pipe", Mode: 0o644, Typeflag: tar.TypeFifo})) + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, rfs.CreateServerFile("bundle.tar.gz", buf.Bytes())) + + require.NoError(t, fs.DecompressFile(context.Background(), "/", "bundle.tar.gz")) + + root := filepath.Join(rfs.root, "server") + target, err := os.Readlink(filepath.Join(root, "link")) + require.NoError(t, err) + assert.Equal(t, "file.txt", target) + for _, name := range []string{"hard", "device", "pipe"} { + _, err := os.Lstat(filepath.Join(root, name)) + assert.ErrorIs(t, err, os.ErrNotExist, name) + } +} + +// A symlink in a zip archive is extracted as a symlink. +func TestDecompressZipSymlinkEntry(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + hdr := &zip.FileHeader{Name: "link"} + hdr.SetMode(iofs.ModeSymlink | 0o777) + w, err := zw.CreateHeader(hdr) + require.NoError(t, err) + _, err = w.Write([]byte("target.txt")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + require.NoError(t, rfs.CreateServerFile("bundle.zip", buf.Bytes())) + + require.NoError(t, fs.DecompressFile(context.Background(), "/", "bundle.zip")) + + target, err := os.Readlink(filepath.Join(rfs.root, "server", "link")) + require.NoError(t, err) + assert.Equal(t, "target.txt", target) +} diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index 9d2003b00..43f564283 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -112,12 +112,13 @@ func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { // If we are now allowing a stale response go ahead and perform the lookup and return the fresh // value. This is a blocking operation to the calling process. if !allowStaleValue { - return fs.updateCachedDiskUsage() - } else if !fs.lookupInProgress.Load() { + return fs.refreshDiskUsage() + } else if fs.lookupInProgress.CompareAndSwap(false, true) { // Otherwise, if we allow a stale value and there isn't a valid item in the cache and we aren't // currently performing a lookup, just do the disk usage calculation in the background. go func(fs *Filesystem) { - if _, err := fs.updateCachedDiskUsage(); err != nil { + defer fs.lookupInProgress.Store(false) + if _, err := fs.refreshDiskUsage(); err != nil { log.WithField("root", fs.Path()).WithField("error", err).Warn("failed to update fs disk usage from within routine") } }(fs) @@ -128,34 +129,71 @@ func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { return fs.unixFS.Usage(), nil } +// refreshDiskUsage updates the currently used disk space for a server, unless +// another lookup finished while this one was waiting for it. +func (fs *Filesystem) refreshDiskUsage() (int64, error) { + fs.lookupMu.Lock() + defer fs.lookupMu.Unlock() + + if fs.lastLookupTime.Get().After(time.Now().Add(time.Second * fs.diskCheckInterval * -1)) { + return fs.unixFS.Usage(), nil + } + return fs.walkDiskUsage() +} + // Updates the currently used disk space for a server. func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { // Obtain an exclusive lock on this process so that we don't unintentionally run it at the same - // time as another running process. Once the lock is available it'll read from the cache for the - // second call rather than hitting the disk in parallel. - fs.mu.Lock() - defer fs.mu.Unlock() + // time as another running process. + fs.lookupMu.Lock() + defer fs.lookupMu.Unlock() + + return fs.walkDiskUsage() +} +// walkDiskUsage walks the disk and updates the cached usage. The caller must +// hold lookupMu. +func (fs *Filesystem) walkDiskUsage() (int64, error) { // Signal that we're currently updating the disk size so that other calls to the disk checking // functions can determine if they should queue up additional calls to this function. Ensure that // we always set this back to "false" when this process is done executing. fs.lookupInProgress.Store(true) defer fs.lookupInProgress.Store(false) + // Writes are not blocked while walking the directory, so keep track of the space reserved for + // writes that are in progress at any point during the walk. + fs.mu.Lock() + fs.walking = true + fs.walkReserved = fs.reserved + fs.mu.Unlock() + // If there is no size its either because there is no data (in which case running this function // will have effectively no impact), or there is nothing in the cache, in which case we need to // grab the size of their data directory. This is a taxing operation, so we want to store it in // the cache once we've gotten it. size, err := fs.DirectorySize("/") - // Always cache the size, even if there is an error. We want to always return that value - // so that we don't cause an endless loop of determining the disk size if there is a temporary - // error encountered. + // Always record the lookup, even if there is an error, so that a persistent error does not + // cause the disk to be walked on every call. fs.lastLookupTime.Set(time.Now()) - fs.unixFS.SetUsage(size) + // The walk may or may not have seen the data written by writes that are still in progress, so + // add the space reserved for them. Otherwise, a write that started before the walk could write + // past the disk limit. Writes that finished during the walk are not added, since the walk has + // usually already counted them, and counting them twice could make the server appear to be + // over its limit. Either error is corrected by the next walk. + fs.mu.Lock() + usage := size + fs.walkReserved + if err != nil { + // A walk that failed part of the way through has only counted some of the files, so + // never lower the usage because of one. + usage = max(usage, fs.unixFS.Usage()) + } + fs.unixFS.SetUsage(usage) + fs.walking = false + fs.mu.Unlock() - return size, err + return usage, err } // DirectorySize calculates the size of a directory and its descendants. @@ -170,6 +208,10 @@ func (fs *Filesystem) DirectorySize(root string) (int64, error) { var size atomic.Int64 err = fs.unixFS.WalkDirat(dirfd, name, func(dirfd int, name, _ string, d ufs.DirEntry, err error) error { + // Files can be removed while the walk is running, which is not an error. + if errors.Is(err, ufs.ErrNotExist) { + return nil + } if err != nil { return errors.Wrap(err, "walkdirat err") } @@ -180,6 +222,9 @@ func (fs *Filesystem) DirectorySize(root string) (int64, error) { } info, err := fs.unixFS.Lstatat(dirfd, name) + if errors.Is(err, ufs.ErrNotExist) { + return nil + } if err != nil { return errors.Wrap(err, "lstatat err") } @@ -208,6 +253,10 @@ func (fs *Filesystem) HasSpaceFor(size int64) error { return nil } +// reserveDisk reserves space for a write that is about to happen, returning an +// error if there is not enough space available. Writes happening at the same time +// therefore cannot each use the space that is left. Every reservation must be +// followed by a call to releaseDisk once the write is finished. func (fs *Filesystem) reserveDisk(size int64) error { if size <= 0 { return nil @@ -220,9 +269,31 @@ func (fs *Filesystem) reserveDisk(size int64) error { return err } fs.unixFS.Add(size) + fs.reserved += size + if fs.walking { + fs.walkReserved += size + } return nil } +// releaseDisk finishes a write that reserved space with reserveDisk, replacing +// the reserved space with the amount the write actually changed the disk usage +// by. +func (fs *Filesystem) releaseDisk(reserved int64, used int64) { + if reserved < 0 { + reserved = 0 + } + + fs.mu.Lock() + defer fs.mu.Unlock() + + fs.reserved -= reserved + if fs.walking { + fs.walkReserved = max(fs.walkReserved-reserved, 0) + } + fs.unixFS.Add(used - reserved) +} + func (fs *Filesystem) adjustDisk(size int64) int64 { if size == 0 { return fs.CachedUsage() @@ -233,8 +304,3 @@ func (fs *Filesystem) adjustDisk(size int64) int64 { return fs.unixFS.Add(size) } - -// Updates the disk usage for the Filesystem instance. -func (fs *Filesystem) addDisk(i int64) int64 { - return fs.unixFS.Add(i) -} diff --git a/server/filesystem/disk_usage_lookup_test.go b/server/filesystem/disk_usage_lookup_test.go new file mode 100644 index 000000000..396578508 --- /dev/null +++ b/server/filesystem/disk_usage_lookup_test.go @@ -0,0 +1,51 @@ +package filesystem + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A walk that fails part of the way through has only counted some of the files, +// so it must not lower the cached usage. +func TestFailedDiskUsageWalkDoesNotLowerUsage(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + fs.unixFS.SetUsage(12345) + require.NoError(t, os.RemoveAll(filepath.Join(rfs.root, "server"))) + + _, err := fs.DiskUsage(false) + require.Error(t, err) + assert.Equal(t, int64(12345), fs.CachedUsage()) +} + +// A caller that found the cache stale and waited for another walk to finish +// uses that walk's result instead of walking the disk again. +func TestQueuedDiskUsageLookupUsesRefreshedCache(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + require.NoError(t, rfs.CreateServerFile("a", randomData(t, 4096))) + + // Hold the lookup lock so that the caller sees a stale cache and waits. + fs.lookupMu.Lock() + done := make(chan error, 1) + go func() { + _, err := fs.DiskUsage(false) + done <- err + }() + time.Sleep(100 * time.Millisecond) + + // While the caller waits, another walk finishes and refreshes the cache. + fs.lastLookupTime.Set(time.Now()) + fs.unixFS.SetUsage(12345) + fs.lookupMu.Unlock() + + require.NoError(t, <-done) + assert.Equal(t, int64(12345), fs.CachedUsage()) +} diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index bab4dc889..17833eb3e 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -2,12 +2,8 @@ package filesystem import ( "fmt" - "path/filepath" "emperror.dev/errors" - "github.com/apex/log" - - "github.com/Rene-Roscher/wings/internal/ufs" ) type ErrorCode string @@ -85,26 +81,6 @@ func (e *Error) Unwrap() error { return e.err } -// Generates an error logger instance with some basic information. -func (fs *Filesystem) error(err error) *log.Entry { - return log.WithField("subsystem", "filesystem").WithField("root", fs.Path()).WithField("error", err) -} - -// Handle errors encountered when walking through directories. -// -// If there is a path resolution error just skip the item entirely. Only return this for a -// directory, otherwise return nil. Returning this error for a file will stop the walking -// for the remainder of the directory. This is assuming an FileInfo struct was even returned. -func (fs *Filesystem) handleWalkerError(err error, f ufs.FileInfo) error { - if !IsErrorCode(err, ErrCodePathResolution) { - return err - } - if f != nil && f.IsDir() { - return filepath.SkipDir - } - return nil -} - // IsFilesystemError checks if the given error is one of the Filesystem errors. func IsFilesystemError(err error) bool { var fserr *Error diff --git a/server/filesystem/extract_transfer_test.go b/server/filesystem/extract_transfer_test.go new file mode 100644 index 000000000..fa96e11ae --- /dev/null +++ b/server/filesystem/extract_transfer_test.go @@ -0,0 +1,43 @@ +package filesystem + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A server being transferred keeps all of its files, including ones the +// denylist would refuse to write. +func TestExtractTransferKeepsDenylistedFiles(t *testing.T) { + // NewFs sets up the configuration the filesystem reads. + _, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + root := filepath.Join(rfs.root, "server") + fs, err := New(root, 0, []string{"denied.txt"}) + require.NoError(t, err) + fs.isTest = true + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range map[string]string{"allowed.txt": "allowed", "denied.txt": "denied"} { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})) + _, err := tw.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gz.Close()) + + require.NoError(t, fs.ExtractTransfer(context.Background(), &buf)) + + for _, name := range []string{"allowed.txt", "denied.txt"} { + _, err := os.Stat(filepath.Join(root, name)) + require.NoError(t, err, name) + } +} diff --git a/server/filesystem/fifo_test.go b/server/filesystem/fifo_test.go new file mode 100644 index 000000000..b12249734 --- /dev/null +++ b/server/filesystem/fifo_test.go @@ -0,0 +1,71 @@ +package filesystem + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// File operations return an error for named pipes rather than opening them. +func TestNamedPipesAreNeverOpened(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + fs.SetDiskLimit(1024 * 1024 * 1024) + + ops := []struct { + name string + op func(p string) error + }{ + {"File", func(p string) error { + f, _, err := fs.File(p) + if err == nil { + _ = f.Close() + } + return err + }}, + {"Touch", func(p string) error { + f, err := fs.Touch(p, ufs.O_RDWR) + if err == nil { + _ = f.Close() + } + return err + }}, + {"Write", func(p string) error { + b := make([]byte, 1<<20) + return fs.Write(p, bytes.NewReader(b), int64(len(b)), 0o644) + }}, + {"Copy", func(p string) error { return fs.Copy(p) }}, + {"DecompressFile", func(p string) error { return fs.DecompressFile(context.Background(), "/", p) }}, + {"SpaceAvailableForDecompression", func(p string) error { + return fs.SpaceAvailableForDecompression(context.Background(), "/", p) + }}, + } + + for _, tt := range ops { + t.Run(tt.name, func(t *testing.T) { + // Opening a pipe for writing unblocks anything waiting to read from it, + // so each operation gets a pipe of its own. + p := "pipe-" + tt.name + if err := unix.Mkfifo(filepath.Join(rfs.root, "server", p), 0o644); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- tt.op(p) }() + select { + case err := <-done: + if err == nil { + t.Fatal("expected an error for a named pipe") + } + case <-time.After(time.Second * 2): + t.Fatal("blocked on a named pipe") + } + }) + } +} diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 06b4486e7..c16afe13a 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -24,7 +24,19 @@ import ( type Filesystem struct { unixFS *ufs.Quota - mu sync.RWMutex + // mu guards changes to the disk usage that must be checked against the + // limit, along with the fields tracking reservations below. + mu sync.RWMutex + // reserved is the space reserved by writes that have not finished yet. + reserved int64 + // walking is set while the disk usage is being calculated, during which + // walkReserved is the space reserved by writes in progress at any point. + walking bool + walkReserved int64 + + // lookupMu prevents the disk usage from being calculated more than once + // at the same time. + lookupMu sync.Mutex lastLookupTime *usageLookupTime lookupInProgress atomic.Bool diskCheckInterval time.Duration @@ -35,7 +47,7 @@ type Filesystem struct { // New creates a new Filesystem instance for a given server. func New(root string, size int64, denylist []string) (*Filesystem, error) { - if err := os.MkdirAll(root, 0o755); err != nil { + if err := os.MkdirAll(root, 0o755); err != nil { //nolint:gosec // mounted as the container's home directory; the parent data directory is 0700 return nil, err } unixFS, err := ufs.NewUnixFS(root, config.UseOpenat2()) @@ -93,19 +105,48 @@ func (fs *Filesystem) UnixFS() *ufs.UnixFS { // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. func (fs *Filesystem) Touch(p string, flag int) (ufs.File, error) { - var currentSize int64 - st, err := fs.unixFS.Stat(p) - if err != nil && !errors.Is(err, ufs.ErrNotExist) { + file, currentSize, err := fs.openForWrite(p, flag, 0o644) + if err != nil { return nil, err - } else if err == nil && !st.IsDir() { - currentSize = st.Size() } + return newQuotaFile(fs, file, currentSize), nil +} - file, err := fs.unixFS.Touch(p, flag, 0o644) +// openForWrite opens p with the given flags, creating it if it does not exist, +// and returns it along with the size it had when it was opened. The size is read +// from the opened file rather than from the path, since the path can be changed, +// for example to a symlink, between being checked and being opened. If flag +// includes O_TRUNC the file is truncated after its size is read. +func (fs *Filesystem) openForWrite(p string, flag int, mode ufs.FileMode) (ufs.File, int64, error) { + file, err := fs.unixFS.Touch(p, flag&^ufs.O_TRUNC, mode) if err != nil { - return nil, err + return nil, 0, err } - return newQuotaFile(fs, file, currentSize), nil + st, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, 0, err + } + var size int64 + if st.Mode().IsRegular() { + size = st.Size() + } + if flag&ufs.O_TRUNC != 0 { + if err := file.Truncate(0); err != nil { + _ = file.Close() + return nil, 0, err + } + } + return file, size, nil +} + +// checkNotDirectory returns an error if p is a directory. +func (fs *Filesystem) checkNotDirectory(p string) error { + if st, err := fs.unixFS.Lstat(p); err == nil && st.IsDir() { + // TODO: resolved + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + } + return nil } // Writefile writes a file to the system. If the file does not already exist one @@ -114,22 +155,14 @@ func (fs *Filesystem) Touch(p string, flag int) (ufs.File, error) { // // DEPRECATED: use `Write` instead. func (fs *Filesystem) Writefile(p string, r io.Reader) error { - var currentSize int64 - st, err := fs.unixFS.Stat(p) - if err != nil && !errors.Is(err, ufs.ErrNotExist) { - return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") - } else if err == nil { - if st.IsDir() { - // TODO: resolved - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) - } - currentSize = st.Size() + if err := fs.checkNotDirectory(p); err != nil { + return err } // Touch the file and return the handle to it at this point. This will // create or truncate the file, and create any necessary parent directories // if they are missing. - file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, 0o644) + file, currentSize, err := fs.openForWrite(p, ufs.O_RDWR|ufs.O_TRUNC, 0o644) if err != nil { return fmt.Errorf("error touching file: %w", err) } @@ -150,22 +183,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { } func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileMode) error { - var currentSize int64 - st, err := fs.unixFS.Stat(p) - if err != nil && !errors.Is(err, ufs.ErrNotExist) { - return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") - } else if err == nil { - if st.IsDir() { - // TODO: resolved - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) - } - currentSize = st.Size() - } - - // Check that the new size we're writing to the disk can fit. If there is currently - // a file we'll subtract that current file size from the size of the buffer to determine - // the amount of new data we're writing (or amount we're removing if smaller). - if err := fs.HasSpaceFor(newSize - currentSize); err != nil { + if err := fs.checkNotDirectory(p); err != nil { return err } @@ -177,28 +195,49 @@ func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileM return err } - // Touch the file and return the handle to it at this point. This will - // create or truncate the file, and create any necessary parent directories - // if they are missing. - file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, mode) + _, statErr := fs.unixFS.Lstat(p) + existed := statErr == nil + + // Open the file before reserving space, so that the size being replaced is the + // size of the file that is written to. + file, currentSize, err := fs.openForWrite(p, ufs.O_RDWR, mode) if err != nil { return err } defer file.Close() - if newSize == 0 { - // Subtract the previous size of the file if the new size is 0. - fs.unixFS.Add(-currentSize) - } else { + // Check that the new size we're writing to the disk can fit, and reserve it so that + // writes happening at the same time cannot each use the space that is left. If there + // is currently a file we'll subtract that current file size from the size of the buffer + // to determine the amount of new data we're writing (or amount we're removing if smaller). + if err := fs.reserveDisk(newSize - currentSize); err != nil { + if !existed { + _ = fs.unixFS.Remove(p) + } + return err + } + reserved := max(newSize-currentSize, 0) + // Release the reservation even if reading r panics, since the panic is + // recovered and the reservation would otherwise last until Wings restarts. + var used int64 + defer func() { fs.releaseDisk(reserved, used) }() + + if err := file.Truncate(0); err != nil { + return err + } + // The file is now empty. + used = -currentSize + + var n int64 + if newSize > 0 { // Do not use CopyBuffer here, it is wasteful as the file implements // io.ReaderFrom, which causes it to not use the buffer anyways. - var n int64 n, err = io.Copy(file, io.LimitReader(r, newSize)) - - // Adjust the disk usage to account for the old size and the new size of the file. - fs.unixFS.Add(n - currentSize) } + // Adjust the disk usage to account for the old size and the new size of the file. + used = n - currentSize + if err := fs.chownFile(p); err != nil { return err } @@ -206,6 +245,37 @@ func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileM return err } +// WriteFrom writes the contents of r to p, creating the file or truncating it +// if it exists, and returns the number of bytes written. Unlike Write, the size +// of the contents does not need to be known ahead of time: each write is counted +// against the disk limit as it happens. If writing fails the file is removed. +func (fs *Filesystem) WriteFrom(p string, r io.Reader) (int64, error) { + if err := fs.checkNotDirectory(p); err != nil { + return 0, err + } + + if err := fs.mkdirAll(filepath.Dir(p), 0o755); err != nil { + return 0, err + } + + file, err := fs.Touch(p, ufs.O_RDWR|ufs.O_TRUNC) + if err != nil { + return 0, err + } + n, err := io.Copy(file, r) + if cerr := file.Close(); err == nil { + err = cerr + } + if err == nil { + err = fs.chownFile(p) + } + if err != nil { + _ = fs.unixFS.Remove(p) + return n, err + } + return n, nil +} + // CreateDirectory creates a new directory (name) at a specified path (p) for // the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { @@ -357,10 +427,14 @@ func (fs *Filesystem) Copy(p string) error { } currentSize := info.Size() - // Check that copying this file wouldn't put the server over its limit. - if err := fs.HasSpaceFor(currentSize); err != nil { + // Check that copying this file wouldn't put the server over its limit, and reserve + // the space so that copies happening at the same time cannot each use it. + if err := fs.reserveDisk(currentSize); err != nil { return err } + // Release the reservation even if the copy panics, see Write. + var used int64 + defer func() { fs.releaseDisk(currentSize, used) }() base := info.Name() extension := filepath.Ext(base) @@ -374,20 +448,30 @@ func (fs *Filesystem) Copy(p string) error { baseName = strings.TrimSuffix(baseName, ".tar") } - newName, err := fs.findCopySuffix(dirfd, baseName, extension) - if err != nil { - return err - } - dst, err := fs.unixFS.OpenFileat(dirfd, newName, ufs.O_WRONLY|ufs.O_CREATE, info.Mode()) - if err != nil { - return err + // Never write into an existing file, such as a copy made by another request + // at the same time. + var ( + newName string + dst ufs.File + ) + for i := 0; ; i++ { + newName, err = fs.findCopySuffix(dirfd, baseName, extension) + if err == nil { + dst, err = fs.unixFS.OpenFileat(dirfd, newName, ufs.O_WRONLY|ufs.O_CREATE|ufs.O_EXCL, info.Mode()) + } + if err == nil { + break + } + if !errors.Is(err, ufs.ErrExist) || i >= 10 { + return err + } } defer dst.Close() // Do not use CopyBuffer here, it is wasteful as the file implements // io.ReaderFrom, which causes it to not use the buffer anyways. n, err := io.Copy(dst, io.LimitReader(source, currentSize)) - fs.unixFS.Add(n) + used = n if !fs.isTest { if err := fs.unixFS.Lchownat(dirfd, newName, config.Get().System.User.Uid, config.Get().System.User.Gid); err != nil { @@ -399,26 +483,19 @@ func (fs *Filesystem) Copy(p string) error { } // TruncateRootDirectory removes _all_ files and directories from a server's -// data directory and resets the used disk space to zero. +// data directory and resets the used disk space to zero. The directory itself +// is kept, since it is mounted into the server's container. func (fs *Filesystem) TruncateRootDirectory() error { - if err := os.RemoveAll(fs.Path()); err != nil { - return err - } - if err := os.Mkdir(fs.Path(), 0o755); err != nil { - return err - } - _ = fs.unixFS.Close() - unixFS, err := ufs.NewUnixFS(fs.Path(), config.UseOpenat2()) - if err != nil { + if err := fs.unixFS.RemoveContents("."); err != nil { return err } - var limit int64 + fs.mu.Lock() + defer fs.mu.Unlock() + // Space reserved by writes that are still in progress remains in use. + fs.unixFS.SetUsage(fs.reserved) if fs.isTest { - limit = 0 - } else { - limit = fs.unixFS.Limit() + fs.unixFS.SetLimit(0) } - fs.unixFS = ufs.NewQuota(unixFS, limit) return nil } diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index afec934da..e0178ca34 100644 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -3,9 +3,9 @@ package filesystem import ( "bufio" "bytes" + "crypto/rand" "errors" "math" - "math/rand" "os" "path/filepath" "testing" @@ -30,7 +30,6 @@ func NewFs() (*Filesystem, *rootFs) { tmpDir, err := os.MkdirTemp(os.TempDir(), "pterodactyl") if err != nil { panic(err) - return nil, nil } rfs := rootFs{root: tmpDir} @@ -38,14 +37,12 @@ func NewFs() (*Filesystem, *rootFs) { p := filepath.Join(tmpDir, "server") if err := os.Mkdir(p, 0o755); err != nil { panic(err) - return nil, nil } fs, _ := New(p, 0, []string{}) fs.isTest = true if err := fs.TruncateRootDirectory(); err != nil { panic(err) - return nil, nil } return fs, &rfs @@ -383,6 +380,7 @@ func TestFilesystem_Rename(t *testing.T) { g.It("does not allow renaming from a location outside the root", func() { err := rfs.CreateServerFileFromString("/../ext-source.txt", "taget content") + g.Assert(err).IsNil() err = fs.Rename("/../ext-source.txt", "target.txt") g.Assert(err).IsNotNil() @@ -461,6 +459,7 @@ func TestFilesystem_Copy(t *testing.T) { g.It("should return an error if the source is outside the root", func() { err := rfs.CreateServerFileFromString("/../ext-source.txt", "text content") + g.Assert(err).IsNil() err = fs.Copy("../ext-source.txt") g.Assert(err).IsNotNil() @@ -566,6 +565,7 @@ func TestFilesystem_Delete(t *testing.T) { g.It("does not delete files outside the root directory", func() { err := rfs.CreateServerFileFromString("/../ext-source.txt", "external content") + g.Assert(err).IsNil() err = fs.Delete("../ext-source.txt") g.Assert(err).IsNotNil() diff --git a/server/filesystem/ignore.go b/server/filesystem/ignore.go new file mode 100644 index 000000000..0817c445d --- /dev/null +++ b/server/filesystem/ignore.go @@ -0,0 +1,166 @@ +package filesystem + +import ( + "fmt" + "strings" + "time" + + ignore "github.com/sabhiram/go-gitignore" +) + +// Limits applied to the gitignore-style pattern lists used by an Archive, whether they +// come from the Panel with a backup request or from a server's .pteroignore file. +const ( + // MaxIgnoreLength is the maximum size, in bytes, of a pattern list. + MaxIgnoreLength = 32 * 1024 + // MaxIgnorePatterns is the maximum number of non-blank, non-comment lines in a list. + MaxIgnorePatterns = 256 + // MaxIgnorePatternWildcards is the maximum number of unescaped "*" characters in a pattern. + MaxIgnorePatternWildcards = 16 +) + +// ValidateIgnore checks a pattern list against the limits above. The returned error +// describes the first limit exceeded and is safe to display to the user. +func ValidateIgnore(s string) error { + _, err := parseIgnore(s) + return err +} + +// compileIgnore validates a pattern list and compiles it into a matcher. +func compileIgnore(s string) (*ignore.GitIgnore, error) { + lines, err := parseIgnore(s) + if err != nil { + return nil, err + } + return ignore.CompileIgnoreLines(lines...), nil +} + +// parseIgnore splits a pattern list into lines, enforces the limits above and escapes any +// regular expression metacharacters that gitignore treats literally so that go-gitignore +// does not interpret them. +func parseIgnore(s string) ([]string, error) { + // The errors below are shown to users as written. + if len(s) > MaxIgnoreLength { + //nolint:staticcheck // ST1005 + return nil, fmt.Errorf("The ignored files list may not be larger than %d KiB.", MaxIgnoreLength/1024) + } + lines := strings.Split(s, "\n") + var patterns int + for n, line := range lines { + if !isIgnorePattern(line) { + continue + } + patterns++ + if patterns > MaxIgnorePatterns { + //nolint:staticcheck // ST1005 + return nil, fmt.Errorf("The ignored files list may not contain more than %d patterns.", MaxIgnorePatterns) + } + escaped, wildcards := escapeIgnorePattern(line) + if wildcards > MaxIgnorePatternWildcards { + //nolint:staticcheck // ST1005 + return nil, fmt.Errorf("The ignored files pattern on line %d may not contain more than %d wildcards.", n+1, MaxIgnorePatternWildcards) + } + lines[n] = escaped + } + return lines, nil +} + +// isIgnorePattern mirrors the checks go-gitignore performs before compiling a line so that +// blank lines and comments are not counted as patterns. +func isIgnorePattern(line string) bool { + line = strings.TrimRight(line, "\r") + if strings.HasPrefix(line, "#") { + return false + } + return strings.Trim(line, " ") != "" +} + +// escapeIgnorePattern escapes the regular expression metacharacters in a pattern that +// gitignore treats literally and returns the number of unescaped wildcards it contains. +func escapeIgnorePattern(line string) (string, int) { + var ( + b strings.Builder + wildcards int + ) + b.Grow(len(line) + 8) + for i := 0; i < len(line); i++ { + c := line[i] + switch c { + case '\\': + // Preserve gitignore escapes such as "\#", "\!" and "\*" as they are. + b.WriteByte(c) + if i+1 < len(line) { + i++ + b.WriteByte(line[i]) + } + case '*': + wildcards++ + b.WriteByte(c) + case '(', ')', '{', '}', '|', '+': + b.WriteByte('\\') + b.WriteByte(c) + default: + b.WriteByte(c) + } + } + return b.String(), wildcards +} + +// The limits above bound the size of a pattern list but not the cost of evaluating it, +// which also grows with the length of every path in the server. Each archive may spend +// ignoreMatchBaseBudget matching patterns plus, for every file evaluated, +// ignoreMatchPerFileBudget and ignoreMatchPerPatternBudget for each pattern in the list, +// so a large server with an ordinary list never trips the budget: an ordinary pattern +// costs around a microsecond per file, far below the budget, whereas the most expensive +// list the limits allow can exceed it. +const ( + ignoreMatchBaseBudget = 5 * time.Second + ignoreMatchPerFileBudget = 250 * time.Microsecond + ignoreMatchPerPatternBudget = 4 * time.Microsecond +) + +// ignoreMatchBudget tracks the time an archive has spent evaluating its ignore patterns. +type ignoreMatchBudget struct { + base time.Duration + perFile time.Duration + spent time.Duration + files int64 +} + +// newIgnoreMatchBudget returns the budget for evaluating a pattern list containing the +// given number of patterns. +func newIgnoreMatchBudget(patterns int) *ignoreMatchBudget { + return &ignoreMatchBudget{ + base: ignoreMatchBaseBudget, + perFile: ignoreMatchPerFileBudget + time.Duration(patterns)*ignoreMatchPerPatternBudget, + } +} + +// ignorePatternCount returns the number of patterns in a pattern list. +func ignorePatternCount(s string) int { + var n int + for _, line := range strings.Split(s, "\n") { + if isIgnorePattern(line) { + n++ + } + } + return n +} + +// track records the time spent evaluating one more file and returns an error, including +// the figures for the logs, once the budget has been exceeded. +func (b *ignoreMatchBudget) track(d time.Duration) error { + b.spent += d + b.files++ + allowed := time.Duration(b.files) * b.perFile + if allowed < b.base { + allowed = b.base + } + if b.spent <= allowed { + return nil + } + return fmt.Errorf( + "filesystem: ignore pattern matching exceeded its time budget: %s spent over %d files (%s per file)", + b.spent.Round(time.Millisecond), b.files, (b.spent / time.Duration(b.files)).Round(time.Microsecond), + ) +} diff --git a/server/filesystem/ignore_test.go b/server/filesystem/ignore_test.go new file mode 100644 index 000000000..0b4e6cbce --- /dev/null +++ b/server/filesystem/ignore_test.go @@ -0,0 +1,163 @@ +package filesystem + +import ( + "strconv" + "strings" + "testing" + "time" + + . "github.com/franela/goblin" +) + +func TestIgnore(t *testing.T) { + g := Goblin(t) + + g.Describe("ValidateIgnore", func() { + g.It("accepts a typical pattern list", func() { + g.Assert(ValidateIgnore("")).IsNil() + g.Assert(ValidateIgnore("# comment\n\nlogs/\n*.log\n**/cache/**\n!important.log\nworld (copy)/\r\n")).IsNil() + }) + + g.It("rejects a list larger than the maximum length", func() { + g.Assert(ValidateIgnore(strings.Repeat("a", MaxIgnoreLength))).IsNil() + g.Assert(ValidateIgnore(strings.Repeat("a", MaxIgnoreLength+1))).IsNotNil() + }) + + g.It("rejects a list with too many patterns", func() { + lines := make([]string, 0, MaxIgnorePatterns+1) + for i := 0; i < MaxIgnorePatterns; i++ { + lines = append(lines, "file") + } + g.Assert(ValidateIgnore(strings.Join(lines, "\n"))).IsNil() + lines = append(lines, "file") + g.Assert(ValidateIgnore(strings.Join(lines, "\n"))).IsNotNil() + }) + + g.It("does not count blank lines or comments as patterns", func() { + lines := make([]string, 0, MaxIgnorePatterns*3) + for i := 0; i < MaxIgnorePatterns; i++ { + lines = append(lines, "file", "", "# comment") + } + g.Assert(ValidateIgnore(strings.Join(lines, "\n"))).IsNil() + }) + + g.It("rejects a pattern with too many wildcards", func() { + g.Assert(ValidateIgnore(strings.Repeat("*a", MaxIgnorePatternWildcards))).IsNil() + g.Assert(ValidateIgnore(strings.Repeat("*a", MaxIgnorePatternWildcards+1))).IsNotNil() + }) + + g.It("does not count escaped wildcards", func() { + g.Assert(ValidateIgnore(strings.Repeat(`\*a`, MaxIgnorePatternWildcards+1))).IsNil() + }) + }) + + g.Describe("escapeIgnorePattern", func() { + g.It("escapes regular expression metacharacters that gitignore treats literally", func() { + escaped, wildcards := escapeIgnorePattern("world (copy)/{a|b}+") + g.Assert(escaped).Equal(`world \(copy\)/\{a\|b\}\+`) + g.Assert(wildcards).Equal(0) + }) + + g.It("preserves gitignore escapes and character classes", func() { + escaped, wildcards := escapeIgnorePattern(`\#not-a-comment\*[^a-z]*.log`) + g.Assert(escaped).Equal(`\#not-a-comment\*[^a-z]*.log`) + g.Assert(wildcards).Equal(1) + }) + }) + + g.Describe("compileIgnore", func() { + g.It("matches paths containing metacharacters literally", func() { + i, err := compileIgnore("world (copy)/\nc++/\n*.log\n!keep.log\n") + g.Assert(err).IsNil() + g.Assert(i.MatchesPath("world (copy)/level.dat")).IsTrue() + g.Assert(i.MatchesPath("world copy/level.dat")).IsFalse() + g.Assert(i.MatchesPath("c++/main.cpp")).IsTrue() + g.Assert(i.MatchesPath("logs/latest.log")).IsTrue() + g.Assert(i.MatchesPath("logs/keep.log")).IsFalse() + g.Assert(i.MatchesPath("server.jar")).IsFalse() + }) + + g.It("does not expand repetition counts", func() { + i, err := compileIgnore("(a{100}){10}\n") + g.Assert(err).IsNil() + g.Assert(i.MatchesPath(strings.Repeat("a", 1000))).IsFalse() + g.Assert(i.MatchesPath("(a{100}){10}")).IsTrue() + }) + + g.It("returns an error for an invalid pattern list", func() { + _, err := compileIgnore(strings.Repeat("*a", MaxIgnorePatternWildcards+1)) + g.Assert(err).IsNotNil() + }) + }) + + g.Describe("ignoreMatchBudget", func() { + g.It("allows the base budget regardless of the file count", func() { + b := &ignoreMatchBudget{base: 10 * time.Millisecond, perFile: time.Millisecond} + g.Assert(b.track(9 * time.Millisecond)).IsNil() + g.Assert(b.track(2 * time.Millisecond)).IsNotNil() + }) + + g.It("scales the budget with the number of files evaluated", func() { + b := &ignoreMatchBudget{base: 10 * time.Millisecond, perFile: time.Millisecond} + for n := 0; n < 100; n++ { + g.Assert(b.track(900 * time.Microsecond)).IsNil() + } + g.Assert(b.track(20 * time.Millisecond)).IsNotNil() + }) + + g.It("reports the figures in the error", func() { + b := &ignoreMatchBudget{base: time.Millisecond, perFile: time.Millisecond} + err := b.track(5 * time.Millisecond) + g.Assert(err).IsNotNil() + g.Assert(strings.Contains(err.Error(), "5ms spent over 1 files (5ms per file)")).IsTrue() + }) + + // The budget for each file is exceeded by the most expensive list allowed, while + // leaving plenty of room for an ordinary one. + g.It("is exceeded by the most expensive list allowed but not by a typical one", func() { + cost := func(lines []string) time.Duration { + i, err := compileIgnore(strings.Join(lines, "\n")) + g.Assert(err).IsNil() + start := time.Now() + for n := 0; n < 20; n++ { + i.MatchesPath("aaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaa") + } + return time.Since(start) / 20 + } + + worst := make([]string, MaxIgnorePatterns) + for n := range worst { + worst[n] = "*" + strings.Repeat("a*", MaxIgnorePatternWildcards-1) + strconv.Itoa(n) + } + worstCost := cost(worst) + if budget := newIgnoreMatchBudget(len(worst)).perFile; worstCost <= budget { + g.Failf("the most expensive list costs %s per file, within the budget of %s", worstCost, budget) + } + + // The longest list of ordinary patterns allowed must fit within its budget. The + // race detector slows matching down by far more than the budget allows for. + if !raceEnabled { + ordinary := make([]string, MaxIgnorePatterns) + for n := range ordinary { + ordinary[n] = []string{"dir" + strconv.Itoa(n) + "/", "*.ext" + strconv.Itoa(n), "plugins/*/cache" + strconv.Itoa(n) + "/"}[n%3] + } + i, err := compileIgnore(strings.Join(ordinary, "\n")) + g.Assert(err).IsNil() + start := time.Now() + for n := 0; n < 20; n++ { + i.MatchesPath("plugins/Essentials/userdata/0123456789abcdef.yml") + } + if c, budget := time.Since(start)/20, newIgnoreMatchBudget(len(ordinary)).perFile; c > budget { + g.Failf("the longest ordinary list costs %s per file, over its budget of %s", c, budget) + } + } + + // Compared against the most expensive list rather than the budget itself, + // since the race detector slows down both by a large amount. + typical := []string{"*.log", "logs/", "cache/", "*.tmp", "world/region/*.mca", "plugins/*/data/", "!plugins/keep/data/"} + if c := cost(typical); c*50 > worstCost { + g.Failf("a typical list costs %s per file, compared to %s for the most expensive list", c, worstCost) + } + }) + }) +} diff --git a/server/filesystem/link_containment_test.go b/server/filesystem/link_containment_test.go new file mode 100644 index 000000000..7c1b448b6 --- /dev/null +++ b/server/filesystem/link_containment_test.go @@ -0,0 +1,42 @@ +package filesystem + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// Writing through a link to a file outside the root fails and leaves that file +// unchanged. +func TestWriteThroughLinkOutsideRootFails(t *testing.T) { + fs, rfs := NewFs() + fs.SetDiskLimit(100) + + target := filepath.Join(rfs.root, "file.bin") + require.NoError(t, os.WriteFile(target, nil, 0o644)) + require.NoError(t, os.Symlink(target, filepath.Join(rfs.root, "server", "link"))) + + err := fs.Write("link", bytes.NewReader(make([]byte, 200)), 200, 0o644) + require.ErrorIs(t, err, ufs.ErrBadPathResolution) + + content, err := os.ReadFile(target) + require.NoError(t, err) + require.Empty(t, content) +} + +// Creating a directory where the name is a link to a directory outside the root +// fails. +func TestCreateDirectoryThroughLinkRefusesOutsideDirectory(t *testing.T) { + fs, rfs := NewFs() + + outsideDir := filepath.Join(rfs.root, "outside-dir") + require.NoError(t, os.Mkdir(outsideDir, 0o755)) + require.NoError(t, os.Symlink(outsideDir, filepath.Join(rfs.root, "server", "dir-link"))) + + require.Error(t, fs.CreateDirectory("dir-link", "/")) +} diff --git a/server/filesystem/norace_test.go b/server/filesystem/norace_test.go new file mode 100644 index 000000000..e8ce08374 --- /dev/null +++ b/server/filesystem/norace_test.go @@ -0,0 +1,7 @@ +//go:build !race + +package filesystem + +// raceEnabled reports whether the race detector is enabled, which slows down +// timing sensitive tests. +const raceEnabled = false diff --git a/server/filesystem/path.go b/server/filesystem/path.go index edb1cad6d..ef5a4efeb 100644 --- a/server/filesystem/path.go +++ b/server/filesystem/path.go @@ -1,6 +1,7 @@ package filesystem import ( + "path" "path/filepath" "strings" @@ -16,28 +17,16 @@ func (fs *Filesystem) IsIgnored(paths ...string) error { // return err //} // TODO: update logic to use unixFS - if fs.denylist.MatchesPath(p) { + // + // Match against the cleaned path the filesystem will use. A trailing slash is + // kept so that it still matches patterns for directories. + cleaned := path.Clean("/" + filepath.ToSlash(p)) + if strings.HasSuffix(p, "/") && cleaned != "/" { + cleaned += "/" + } + if fs.denylist.MatchesPath(cleaned) { return errors.WithStack(&Error{code: ErrCodeDenylistFile, path: p, resolved: p}) } } return nil } - -// Generate a path to the file by cleaning it up and appending the root server path to it. This -// DOES NOT guarantee that the file resolves within the server data directory. You'll want to use -// the fs.unsafeIsInDataDirectory(p) function to confirm. -func (fs *Filesystem) unsafeFilePath(p string) string { - // Calling filepath.Clean on the joined directory will resolve it to the absolute path, - // removing any ../ type of resolution arguments, and leaving us with a direct path link. - // - // This will also trim the existing root path off the beginning of the path passed to - // the function since that can get a bit messy. - return filepath.Clean(filepath.Join(fs.Path(), strings.TrimPrefix(p, fs.Path()))) -} - -// Check that that path string starts with the server data directory path. This function DOES NOT -// validate that the rest of the path does not end up resolving out of this directory, or that the -// targeted file or folder is not a symlink doing the same thing. -func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool { - return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/") -} diff --git a/server/filesystem/path_denylist_test.go b/server/filesystem/path_denylist_test.go new file mode 100644 index 000000000..2704240be --- /dev/null +++ b/server/filesystem/path_denylist_test.go @@ -0,0 +1,28 @@ +package filesystem + +import ( + "testing" +) + +func TestIsIgnoredMatchesTheResolvedPath(t *testing.T) { + NewFs() + fs, err := New(t.TempDir(), 0, []string{"/server.jar", "config/"}) + if err != nil { + t.Fatal(err) + } + + for _, p := range []string{ + "server.jar", "/server.jar", "./server.jar", "/./server.jar", "//server.jar", + "/sub/../server.jar", "../server.jar", "config/a.yml", "/x/../config/a.yml", + "/config/", "config/", "/x/../config/", + } { + if err := fs.IsIgnored(p); !IsErrorCode(err, ErrCodeDenylistFile) { + t.Errorf("expected %q to be denied, got %v", p, err) + } + } + for _, p := range []string{"sub/server.jar", "/server.jar.bak", "/configs/a.yml"} { + if err := fs.IsIgnored(p); err != nil { + t.Errorf("expected %q to be allowed, got %v", p, err) + } + } +} diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go index ef5d86ea0..cbaf37f47 100644 --- a/server/filesystem/path_test.go +++ b/server/filesystem/path_test.go @@ -139,7 +139,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err = fs.Rename("my_file.txt", "foo/my_file.txt") g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue() - st, err = os.Lstat(filepath.Join(rfs.root, "malicious_dir", "my_file.txt")) + _, err = os.Lstat(filepath.Join(rfs.root, "malicious_dir", "my_file.txt")) g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue() }) }) diff --git a/server/filesystem/quota_file.go b/server/filesystem/quota_file.go index ce924de1b..434ecdecd 100644 --- a/server/filesystem/quota_file.go +++ b/server/filesystem/quota_file.go @@ -57,8 +57,9 @@ func (f *quotaFile) writeAtLocked(p []byte, off int64, write func() (int, error) if !ok { return 0, newFilesystemError(ErrCodeDiskSpace, nil) } - if growth := end - previousSize; growth > 0 { - if err := f.fs.reserveDisk(growth); err != nil { + reserved := end - previousSize + if reserved > 0 { + if err := f.fs.reserveDisk(reserved); err != nil { return 0, err } } @@ -72,19 +73,28 @@ func (f *quotaFile) writeAtLocked(p []byte, off int64, write func() (int, error) } } - if reserved := end - previousSize; reserved > 0 { + if reserved > 0 { actual := int64(0) if writtenEnd > previousSize { actual = writtenEnd - previousSize } - if actual < reserved { - f.fs.adjustDisk(actual - reserved) - } + f.fs.releaseDisk(reserved, actual) } return n, err } +// CheckSize returns an error if growing the file to size bytes would exceed the +// disk limit. +func (f *quotaFile) CheckSize(size int64) error { + f.mu.Lock() + defer f.mu.Unlock() + if size <= f.size { + return nil + } + return f.fs.HasSpaceFor(size - f.size) +} + func quotaWriteEnd(off int64, size int) (int64, bool) { if size < 0 || off > math.MaxInt64-int64(size) { return 0, false diff --git a/server/filesystem/quota_links_test.go b/server/filesystem/quota_links_test.go new file mode 100644 index 000000000..a9376e52c --- /dev/null +++ b/server/filesystem/quota_links_test.go @@ -0,0 +1,82 @@ +package filesystem + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The disk usage counts a file once however many names it has, so deleting one +// of its names does not free any space. +func TestDeletingLinkedNamesKeepsUsageAtSizeOnDisk(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const size = 8 << 10 + require.NoError(t, rfs.CreateServerFile("a", randomData(t, size))) + base := filepath.Join(rfs.root, "server") + for _, name := range []string{"b", "c", "d"} { + require.NoError(t, os.Link(filepath.Join(base, "a"), filepath.Join(base, name))) + } + + _, err := fs.DiskUsage(false) + require.NoError(t, err) + require.Equal(t, int64(size), fs.CachedUsage()) + + for _, name := range []string{"b", "c", "d"} { + require.NoError(t, fs.Delete(name)) + } + assert.Equal(t, int64(size), fs.CachedUsage()) + + // Removing the last name frees the space. + require.NoError(t, fs.Delete("a")) + assert.Equal(t, int64(0), fs.CachedUsage()) +} + +func TestDeletingLinkedNamesInDirectoryKeepsUsageAtSizeOnDisk(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const size = 8 << 10 + require.NoError(t, rfs.CreateServerFile("a", randomData(t, size))) + base := filepath.Join(rfs.root, "server") + require.NoError(t, os.Mkdir(filepath.Join(base, "dir"), 0o755)) + for _, name := range []string{"b", "c"} { + require.NoError(t, os.Link(filepath.Join(base, "a"), filepath.Join(base, "dir", name))) + } + + _, err := fs.DiskUsage(false) + require.NoError(t, err) + + require.NoError(t, fs.Delete("dir")) + assert.Equal(t, int64(size), fs.CachedUsage()) + assert.Equal(t, diskUsage(t, rfs), fs.CachedUsage()) +} + +// After deleting linked names, a write that does not fit the space left on disk +// is still refused. +func TestDeletingLinkedNamesDoesNotAllowWritesOverLimit(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const size = 8 << 10 + require.NoError(t, rfs.CreateServerFile("a", randomData(t, size))) + base := filepath.Join(rfs.root, "server") + for _, name := range []string{"b", "c"} { + require.NoError(t, os.Link(filepath.Join(base, "a"), filepath.Join(base, name))) + } + + _, err := fs.DiskUsage(false) + require.NoError(t, err) + for _, name := range []string{"b", "c"} { + require.NoError(t, fs.Delete(name)) + } + + fs.SetDiskLimit(2 * size) + err = fs.Write("new", bytes.NewReader(randomData(t, 2*size)), 2*size, 0o644) + assert.True(t, IsErrorCode(err, ErrCodeDiskSpace), "expected the write to be refused, got %v", err) +} diff --git a/server/filesystem/quota_test.go b/server/filesystem/quota_test.go new file mode 100644 index 000000000..b31170916 --- /dev/null +++ b/server/filesystem/quota_test.go @@ -0,0 +1,290 @@ +package filesystem + +import ( + "bytes" + "crypto/rand" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// diskUsage returns the actual size of the files in the server's directory. +func diskUsage(t *testing.T, rfs *rootFs) int64 { + t.Helper() + var size int64 + err := filepath.Walk(filepath.Join(rfs.root, "server"), func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsRegular() { + size += info.Size() + } + return nil + }) + if err != nil { + t.Fatal(err) + } + return size +} + +func randomData(t *testing.T, n int) []byte { + t.Helper() + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + return b +} + +// limitWithRoomFor sets the disk limit to the current usage plus the given number +// of bytes, returning the limit. +func limitWithRoomFor(t *testing.T, fs *Filesystem, n int64) int64 { + t.Helper() + usage, err := fs.DiskUsage(false) + if err != nil { + t.Fatal(err) + } + fs.SetDiskLimit(usage + n) + return usage + n +} + +func TestParallelArchivesRespectDiskLimit(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const count = 8 + for i := 0; i < count; i++ { + dir := fmt.Sprintf("dir%d", i) + if err := os.MkdirAll(filepath.Join(rfs.root, "server", dir), 0o755); err != nil { + t.Fatal(err) + } + if err := rfs.CreateServerFile(filepath.Join(dir, "data.bin"), randomData(t, 1<<20)); err != nil { + t.Fatal(err) + } + } + // Enough room for one more archive of the (incompressible) data, but not two. + limit := limitWithRoomFor(t, fs, 1<<20+512<<10) + + var wg sync.WaitGroup + for i := 0; i < count; i++ { + wg.Add(1) + go func(dir string) { + defer wg.Done() + _, _ = fs.CompressFiles(dir, []string{"data.bin"}) + }(fmt.Sprintf("/dir%d", i)) + } + wg.Wait() + + if used := diskUsage(t, rfs); used > limit { + t.Fatalf("parallel archives used %d bytes, exceeding the disk limit of %d bytes", used, limit) + } + if used, tracked := diskUsage(t, rfs), fs.CachedUsage(); used != tracked { + t.Fatalf("tracked disk usage %d does not match the actual usage %d", tracked, used) + } +} + +func TestParallelCopiesRespectDiskLimit(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + if err := rfs.CreateServerFile("data.bin", randomData(t, 1<<20)); err != nil { + t.Fatal(err) + } + limit := limitWithRoomFor(t, fs, 1<<20+512<<10) + + const count = 8 + var wg sync.WaitGroup + var copied atomic.Int64 + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if fs.Copy("data.bin") == nil { + copied.Add(1) + } + }() + } + wg.Wait() + + if used := diskUsage(t, rfs); used > limit { + t.Fatalf("parallel copies used %d bytes, exceeding the disk limit of %d bytes", used, limit) + } + if used, tracked := diskUsage(t, rfs), fs.CachedUsage(); used != tracked { + t.Fatalf("tracked disk usage %d does not match the actual usage %d", tracked, used) + } + entries, err := os.ReadDir(filepath.Join(rfs.root, "server")) + if err != nil { + t.Fatal(err) + } + if int64(len(entries)) != 1+copied.Load() { + t.Fatalf("expected %d files, got %d: copies were written into the same file", 1+copied.Load(), len(entries)) + } +} + +// Recalculating the disk usage while a write is in progress must not lose the +// space reserved for it, otherwise the next write could use that space as well. +func TestDiskUsageRecalculationKeepsReservations(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + limit := limitWithRoomFor(t, fs, 1<<20+512<<10) + + data := randomData(t, 1<<20) + r, w := io.Pipe() + done := make(chan error, 1) + go func() { + done <- fs.Write("slow.bin", r, int64(len(data)), 0o644) + }() + // Send half of the file, then recalculate the disk usage while the rest of it + // is still on its way. + if _, err := w.Write(data[:len(data)/2]); err != nil { + t.Fatal(err) + } + if _, err := fs.updateCachedDiskUsage(); err != nil { + t.Fatal(err) + } + + err := fs.Write("second.bin", bytes.NewReader(data), int64(len(data)), 0o644) + if !IsErrorCode(err, ErrCodeDiskSpace) { + t.Fatalf("expected the second write to be refused, got %v", err) + } + + if _, err := w.Write(data[len(data)/2:]); err != nil { + t.Fatal(err) + } + _ = w.Close() + if err := <-done; err != nil { + t.Fatal(err) + } + if used := diskUsage(t, rfs); used > limit { + t.Fatalf("writes used %d bytes, exceeding the disk limit of %d bytes", used, limit) + } +} + +func TestParallelWritesRespectDiskLimit(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + if err := rfs.CreateServerFile("existing.bin", randomData(t, 1<<20)); err != nil { + t.Fatal(err) + } + limit := limitWithRoomFor(t, fs, 1<<20+512<<10) + + const count = 8 + data := randomData(t, 1<<20) + var wg sync.WaitGroup + for i := 0; i < count; i++ { + wg.Add(1) + go func(p string) { + defer wg.Done() + _ = fs.Write(p, bytes.NewReader(data), int64(len(data)), 0o644) + }(fmt.Sprintf("file%d.bin", i)) + } + wg.Wait() + + if used := diskUsage(t, rfs); used > limit { + t.Fatalf("parallel writes used %d bytes, exceeding the disk limit of %d bytes", used, limit) + } + if used, tracked := diskUsage(t, rfs), fs.CachedUsage(); used != tracked { + t.Fatalf("tracked disk usage %d does not match the actual usage %d", tracked, used) + } +} + +// Writes that finish while the disk usage is being recalculated were already +// counted by the walk, so they must not be counted again. +func TestWritesFinishedDuringDiskUsageWalkAreCountedOnce(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + root := filepath.Join(rfs.root, "server") + + // A directory with many files keeps the walk busy while the write happens. + if err := os.MkdirAll(filepath.Join(root, "slow"), 0o755); err != nil { + t.Fatal(err) + } + for i := 0; i < 100000; i++ { + if err := os.WriteFile(filepath.Join(root, "slow", fmt.Sprintf("f%d", i)), nil, 0o644); err != nil { + t.Fatal(err) + } + } + // Write to a file the walk only reaches after the slow directory, so that the + // walk counts all of it. + for i := 0; i < 8; i++ { + if err := os.WriteFile(filepath.Join(root, fmt.Sprintf("z%d.bin", i)), nil, 0o644); err != nil { + t.Fatal(err) + } + } + d, err := os.Open(root) + if err != nil { + t.Fatal(err) + } + names, _ := d.Readdirnames(-1) + _ = d.Close() + var target string + for i, n := range names { + if n == "slow" && i+1 < len(names) { + target = names[i+1] + } + } + if target == "" { + t.Skip("no file is walked after the slow directory") + } + fs.SetDiskLimit(1 << 30) + if _, err := fs.updateCachedDiskUsage(); err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + defer close(done) + _, _ = fs.updateCachedDiskUsage() + }() + for !fs.lookupInProgress.Load() { + } + // Write through Touch as SFTP does, which reserves space for every chunk. + f, err := fs.Touch(target, ufs.O_RDWR|ufs.O_TRUNC) + if err != nil { + t.Fatal(err) + } + chunk := bytes.Repeat([]byte{1}, 32<<10) + for n := 0; n < 16<<20; n += len(chunk) { + if _, err := f.Write(chunk); err != nil { + t.Fatal(err) + } + } + _ = f.Close() + if !fs.lookupInProgress.Load() { + t.Skip("the walk finished before the write did") + } + <-done + + if used, tracked := diskUsage(t, rfs), fs.CachedUsage(); used != tracked { + t.Fatalf("tracked disk usage %d does not match the actual usage %d", tracked, used) + } +} + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("read failed") } + +// A panic while writing is recovered by the HTTP server, so the space reserved +// for the write must still be released. +func TestWriteReleasesReservedSpaceWhenItPanics(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + limitWithRoomFor(t, fs, 1<<20) + + func() { + defer func() { _ = recover() }() + _ = fs.Write("panic.bin", panicReader{}, 1<<20, 0o644) + }() + + if err := fs.Write("after.bin", bytes.NewReader(make([]byte, 1<<20)), 1<<20, 0o644); err != nil { + t.Fatalf("expected the space reserved by the write that panicked to be released, got %v", err) + } +} diff --git a/server/filesystem/race_test.go b/server/filesystem/race_test.go new file mode 100644 index 000000000..dc7ecbd6b --- /dev/null +++ b/server/filesystem/race_test.go @@ -0,0 +1,7 @@ +//go:build race + +package filesystem + +// raceEnabled reports whether the race detector is enabled, which slows down +// timing sensitive tests. +const raceEnabled = true diff --git a/server/filesystem/write_links_test.go b/server/filesystem/write_links_test.go new file mode 100644 index 000000000..b895a2d50 --- /dev/null +++ b/server/filesystem/write_links_test.go @@ -0,0 +1,41 @@ +package filesystem + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/internal/ufs" +) + +// openForWrite returns the size the opened file had, and only truncates it when +// asked to. +func TestOpenForWriteReturnsOpenedFileSize(t *testing.T) { + fs, rfs := NewFs() + t.Cleanup(func() { _ = os.RemoveAll(rfs.root) }) + + const size = 4096 + require.NoError(t, rfs.CreateServerFile("a.bin", randomData(t, size))) + + f, got, err := fs.openForWrite("a.bin", ufs.O_RDWR, 0o644) + require.NoError(t, err) + require.Equal(t, int64(size), got) + st, err := f.Stat() + require.NoError(t, err) + require.Equal(t, int64(size), st.Size()) + require.NoError(t, f.Close()) + + f, got, err = fs.openForWrite("a.bin", ufs.O_RDWR|ufs.O_TRUNC, 0o644) + require.NoError(t, err) + require.Equal(t, int64(size), got) + st, err = f.Stat() + require.NoError(t, err) + require.Zero(t, st.Size()) + require.NoError(t, f.Close()) + + f, got, err = fs.openForWrite("new.bin", ufs.O_RDWR, 0o644) + require.NoError(t, err) + require.Zero(t, got) + require.NoError(t, f.Close()) +} diff --git a/server/install.go b/server/install.go index 64054b4ab..21582ac06 100644 --- a/server/install.go +++ b/server/install.go @@ -3,16 +3,17 @@ package server import ( "bufio" "context" - "html/template" "io" "os" "path/filepath" "strconv" "strings" + "text/template" "time" "emperror.dev/errors" "github.com/apex/log" + cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" @@ -162,6 +163,16 @@ func (s *Server) SetTransferring(state bool) { } } +// StartTransferring marks the server as being transferred, returning false if +// it already is, or if it is being installed, restored or backed up. +func (s *Server) StartTransferring() bool { + if s.IsInstalling() || s.IsRestoring() || s.IsBackingUp() || !s.transferring.SwapIf(true) { + return false + } + s.Sftp().CancelAll() + return true +} + func (s *Server) IsRestoring() bool { return s.restoring.Load() } @@ -191,7 +202,7 @@ func (ip *InstallationProcess) RemoveContainer() error { RemoveVolumes: true, Force: true, }) - if err != nil && !client.IsErrNotFound(err) { + if err != nil && !cerrdefs.IsNotFound(err) { return err } return nil @@ -248,7 +259,7 @@ func (ip *InstallationProcess) writeScriptToDisk() error { if err := os.MkdirAll(ip.tempDir(), 0o700); err != nil { return errors.WithMessage(err, "could not create temporary directory for install process") } - f, err := os.OpenFile(filepath.Join(ip.tempDir(), "install.sh"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) + f, err := os.OpenFile(filepath.Join(ip.tempDir(), "install.sh"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) //nolint:gosec // read by the install container, which may not run as root if err != nil { return errors.WithMessage(err, "failed to write server installation script to disk before mount") } @@ -349,7 +360,11 @@ func (ip *InstallationProcess) GetLogPath() string { // This grabs the logs from the process to store in the server configuration // directory, and then destroys the associated installation container. func (ip *InstallationProcess) AfterExecute(containerId string) error { - defer ip.RemoveContainer() + defer func() { + if err := ip.RemoveContainer(); err != nil { + ip.Server.Log().WithField("error", err).Warn("failed to remove installation container") + } + }() ip.Server.Log().WithField("container_id", containerId).Debug("pulling installation logs for server") reader, err := ip.client.ContainerLogs(ip.Server.Context(), containerId, container.LogsOptions{ @@ -358,7 +373,7 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { Follow: false, }) - if err != nil && !client.IsErrNotFound(err) { + if err != nil && !cerrdefs.IsNotFound(err) { return err } @@ -443,10 +458,12 @@ func (ip *InstallationProcess) Execute() (string, error) { ReadOnly: false, }, { + // The script only needs to be read, and must not be changed while + // it is running. Target: "/mnt/install", Source: ip.tempDir(), Type: mount.TypeBind, - ReadOnly: false, + ReadOnly: true, }, }, Resources: ip.resourceLimits(), @@ -457,6 +474,12 @@ func (ip *InstallationProcess) Execute() (string, error) { LogConfig: cfg.Docker.ContainerLogConfig(), NetworkMode: container.NetworkMode(cfg.Docker.Network.Mode), UsernsMode: container.UsernsMode(cfg.Docker.UsernsMode), + // Install scripts run as root, and commonly install packages, so they keep + // the capabilities that needs. Drop the ones they do not need. + SecurityOpt: []string{"no-new-privileges"}, + CapDrop: []string{ + "setpcap", "mknod", "audit_write", "net_raw", "sys_chroot", + }, } // Ensure the root directory for the server exists properly before attempting @@ -563,14 +586,18 @@ func (ip *InstallationProcess) resourceLimits() container.Resources { } resources := cfg.AsContainerResources() - // Explicitly remove the PID limits for the installation container. These scripts are - // defined at an administrative level and users can't manually execute things like a - // fork bomb during this process. - resources.PidsLimit = nil + // Installation scripts can need more processes than the server itself, for example + // when compiling, so use a higher limit than the server's. + pids := max(config.Get().Docker.ContainerPidLimit, installerPidLimit) + resources.PidsLimit = &pids return resources } +// installerPidLimit is the lowest limit on the number of processes in an +// installation container. +const installerPidLimit int64 = 4096 + // SyncInstallState makes an HTTP request to the Panel instance notifying it that // the server has completed the installation process, and what the state of the // server is. diff --git a/server/install_container_test.go b/server/install_container_test.go new file mode 100644 index 000000000..aa4454461 --- /dev/null +++ b/server/install_container_test.go @@ -0,0 +1,159 @@ +package server + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/docker/docker/api/types/container" + dockerclient "github.com/docker/docker/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" + "github.com/Rene-Roscher/wings/server/filesystem" +) + +const installTestUUID = "5c7a1b9e-2f40-4d8a-9b3e-7a1c2d3e4f50" + +// installerCreateRequest is the body of a container create call. The Config +// fields are sent at the top level and the HostConfig under its own key. +type installerCreateRequest struct { + container.Config + HostConfig *container.HostConfig `json:"HostConfig"` +} + +// fakeInstallerDocker answers the Docker API calls made by the installer code +// path. It records the create body, fails the start call so Execute returns +// without a real container, and serves a fixed log body. +type fakeInstallerDocker struct { + mu sync.Mutex + create *installerCreateRequest + logs string +} + +func (f *fakeInstallerDocker) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/containers/create"): + var req installerCreateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + f.create = &req + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"Id":"installer-test-id","Warnings":[]}`)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/start"): + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"message":"stopped by test after container create"}`)) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/logs"): + _, _ = io.WriteString(w, f.logs) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } +} + +// newInstallerTestProcess builds an installation process whose Docker client +// talks to a fake API server, with a data directory, temp directory and log +// directory that are all private to the test. +func newInstallerTestProcess(t *testing.T, fake *fakeInstallerDocker) *InstallationProcess { + t.Helper() + + srv := httptest.NewServer(fake) + t.Cleanup(srv.Close) + + c, err := dockerclient.NewClientWithOpts(dockerclient.WithHost(strings.Replace(srv.URL, "http://", "tcp://", 1))) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + require.NoError(t, err) + cfg.AuthenticationToken = "test-token" + cfg.System.TmpDirectory = t.TempDir() + cfg.System.LogDirectory = t.TempDir() + config.Set(cfg) + + s, err := New(nil) + require.NoError(t, err) + s.cfg.Uuid = installTestUUID + s.fs, err = filesystem.New(filepath.Join(t.TempDir(), installTestUUID), 0, nil) + require.NoError(t, err) + + return &InstallationProcess{ + Server: s, + Script: &remote.InstallationScript{ + ContainerImage: "ghcr.io/pterodactyl/installers:debian", + Entrypoint: "bash", + Script: "echo installing", + }, + client: c, + } +} + +// captureInstallerCreate runs the installer until its container create call +// and returns the HostConfig and Config that were sent to Docker. +func captureInstallerCreate(t *testing.T) (*container.HostConfig, *container.Config) { + t.Helper() + + fake := &fakeInstallerDocker{} + p := newInstallerTestProcess(t, fake) + + _, err := p.Execute() + require.Error(t, err, "the fake daemon fails the start call on purpose") + + fake.mu.Lock() + defer fake.mu.Unlock() + require.NotNil(t, fake.create, "installer never sent a container create request") + require.NotNil(t, fake.create.HostConfig) + return fake.create.HostConfig, &fake.create.Config +} + +// The install container runs without the privileges installation scripts do +// not need. +func TestInstallContainerIsRestricted(t *testing.T) { + hc, _ := captureInstallerCreate(t) + + // The Docker client sends capabilities by their full names. + for _, capability := range []string{"CAP_NET_RAW", "CAP_MKNOD", "CAP_SETPCAP", "CAP_SYS_CHROOT"} { + assert.Contains(t, hc.CapDrop, capability) + } + assert.Contains(t, hc.SecurityOpt, "no-new-privileges") + + require.NotNil(t, hc.Resources.PidsLimit) + assert.GreaterOrEqual(t, *hc.Resources.PidsLimit, installerPidLimit) + + for _, m := range hc.Mounts { + if m.Target == "/mnt/install" { + assert.True(t, m.ReadOnly, "the installation script should be mounted read-only") + } + } +} + +// The install log records each environment variable exactly as it was passed to +// the container. +func TestInstallLogKeepsEnvironmentValuesVerbatim(t *testing.T) { + fake := &fakeInstallerDocker{logs: "installer output\n"} + p := newInstallerTestProcess(t, fake) + p.Server.cfg.EnvVars = environment.Variables{"DB_PASSWORD": "p&ss'word<1>"} + + require.NoError(t, os.MkdirAll(filepath.Join(config.Get().System.LogDirectory, "install"), 0o700)) + require.NoError(t, p.AfterExecute("installer-test-id")) + + body, err := os.ReadFile(p.GetLogPath()) + require.NoError(t, err) + assert.Contains(t, string(body), "DB_PASSWORD=p&ss'word<1>") +} diff --git a/server/installer/installer.go b/server/installer/installer.go index 2cc492a17..059c65af3 100644 --- a/server/installer/installer.go +++ b/server/installer/installer.go @@ -38,7 +38,7 @@ func New(ctx context.Context, manager *server.Manager, details ServerDetails) (* // Create a new server instance using the configuration we wrote to the disk // so that everything gets instantiated correctly on the struct. - s, err := manager.InitServer(c) + s, err := manager.InitServer(details.UUID, c) if err != nil { return nil, errors.WrapIf(err, "installer: could not init server instance") } diff --git a/server/listeners.go b/server/listeners.go index 01f2fa37c..6cba03905 100644 --- a/server/listeners.go +++ b/server/listeners.go @@ -1,7 +1,6 @@ package server import ( - "bytes" "regexp" "strconv" "sync" @@ -13,15 +12,8 @@ import ( "github.com/Rene-Roscher/wings/system" "github.com/Rene-Roscher/wings/environment" - "github.com/Rene-Roscher/wings/remote" ) -var dockerEvents = []string{ - environment.DockerImagePullStatus, - environment.DockerImagePullStarted, - environment.DockerImagePullCompleted, -} - type diskSpaceLimiter struct { o sync.Once mu sync.Mutex @@ -59,10 +51,10 @@ func (dsl *diskSpaceLimiter) Trigger() { // output lines to determine if the server is started yet, and if the output is // not being throttled, will send the data over to the websocket. func (s *Server) processConsoleOutputEvent(v []byte) { - // Always process the console output, but do this in a seperate thread since we - // don't really care about side-effects from this call, and don't want it to block - // the console sending logic. - go s.onConsoleOutput(v) + // Always check the console output, including lines that are throttled, so that + // the line marking the server as started is not missed. Lines are checked in + // the order they are read. + s.onConsoleOutput(v) // If the console is being throttled, do nothing else with it, we don't want // to waste time. This code previously terminated server instances after violating @@ -184,14 +176,7 @@ func (s *Server) onConsoleOutput(data []byte) { } } - // If the command sent to the server is one that should stop the server we will need to - // set the server to be in a stopping state, otherwise crash detection will kick in and - // cause the server to unexpectedly restart on the user. - if s.IsRunning() { - stop := processConfiguration.Stop - - if stop.Type == remote.ProcessStopCommand && bytes.Equal(v, []byte(stop.Value)) { - s.Environment.SetState(environment.ProcessOfflineState) - } - } + // Output from the server process is not used to mark it as stopping or stopped. + // The environment marks the server as stopping when the stop command is sent to + // it. } diff --git a/server/listeners_test.go b/server/listeners_test.go new file mode 100644 index 000000000..50850ebba --- /dev/null +++ b/server/listeners_test.go @@ -0,0 +1,55 @@ +package server + +import ( + "sync" + "testing" + + "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/remote" +) + +// stateEnvironment is an environment that records the states it is set to. +type stateEnvironment struct { + environment.ProcessEnvironment + mu sync.Mutex + state string + states []string +} + +func (e *stateEnvironment) State() string { + e.mu.Lock() + defer e.mu.Unlock() + return e.state +} + +func (e *stateEnvironment) SetState(st string) { + e.mu.Lock() + defer e.mu.Unlock() + e.state = st + e.states = append(e.states, st) +} + +func TestConsoleOutputDoesNotChangeState(t *testing.T) { + for _, st := range []string{environment.ProcessStartingState, environment.ProcessRunningState} { + t.Run(st, func(t *testing.T) { + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + env := &stateEnvironment{state: st} + s.Environment = env + s.procConfig = &remote.ProcessConfiguration{} + s.procConfig.Stop.Type = remote.ProcessStopCommand + s.procConfig.Stop.Value = "stop" + + // Output that matches the stop command. + for i := 0; i < 3; i++ { + s.onConsoleOutput([]byte("stop")) + } + + if env.State() != st { + t.Fatalf("expected server to remain %s, got %s (transitions: %v)", st, env.State(), env.states) + } + }) + } +} diff --git a/server/manager.go b/server/manager.go index 7d4323928..95a2ffaac 100644 --- a/server/manager.go +++ b/server/manager.go @@ -3,7 +3,6 @@ package server import ( "context" "encoding/json" - "fmt" "io" "os" "path/filepath" @@ -92,6 +91,20 @@ func (m *Manager) Add(s *Server) { m.mu.Unlock() } +// AddIfMissing adds an item to the collection store unless a server with the +// same ID is already present, returning false if it was not added. +func (m *Manager) AddIfMissing(s *Server) bool { + m.mu.Lock() + defer m.mu.Unlock() + for _, v := range m.servers { + if v.ID() == s.ID() { + return false + } + } + m.servers = append(m.servers, s) + return true +} + // Get returns a single server instance and a boolean value indicating if it was // found in the global collection or not. func (m *Manager) Get(uuid string) (*Server, bool) { @@ -154,7 +167,7 @@ func (m *Manager) PersistStates() error { if err != nil { return errors.WithStack(err) } - if err := os.WriteFile(config.Get().System.GetStatesPath(), data, 0o644); err != nil { + if err := os.WriteFile(config.Get().System.GetStatesPath(), data, 0o600); err != nil { return errors.WithStack(err) } return nil @@ -162,7 +175,7 @@ func (m *Manager) PersistStates() error { // ReadStates returns the state of the servers. func (m *Manager) ReadStates() (map[string]string, error) { - f, err := os.OpenFile(config.Get().System.GetStatesPath(), os.O_RDONLY|os.O_CREATE, 0o644) + f, err := os.OpenFile(config.Get().System.GetStatesPath(), os.O_RDONLY|os.O_CREATE, 0o600) if err != nil { return nil, errors.WithStack(err) } @@ -181,10 +194,10 @@ func (m *Manager) ReadStates() (map[string]string, error) { return out, nil } -// InitServer initializes a server using a data byte array. This will be -// marshaled into the given struct using a YAML marshaler. This will also -// configure the given environment for a server. -func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, error) { +// InitServer initializes the server with the given uuid using a data byte +// array. This will be marshaled into the given struct using a YAML marshaler. +// This will also configure the given environment for a server. +func (m *Manager) InitServer(uuid string, data remote.ServerConfigurationResponse) (*Server, error) { s, err := New(m.client) if err != nil { return nil, err @@ -195,6 +208,9 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, if err := s.SyncWithConfiguration(data); err != nil { return nil, errors.WithStackIf(err) } + if s.ID() != uuid { + return nil, errors.New("server: configuration returned by the Panel is for a different server") + } s.fs, err = filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) if err != nil { @@ -262,12 +278,15 @@ func (m *Manager) init(ctx context.Context) error { log.WithField("server", data.Uuid).WithField("error", err).Error("failed to parse server configuration from API response, skipping...") return } - s, err := m.InitServer(d) + s, err := m.InitServer(data.Uuid, d) if err != nil { log.WithField("server", data.Uuid).WithField("error", err).Error("failed to load server, skipping...") return } - m.Add(s) + if !m.AddIfMissing(s) { + s.CleanupForDestroy() + log.WithField("server", data.Uuid).Error("server was returned by the API more than once, skipping...") + } }) } @@ -275,8 +294,8 @@ func (m *Manager) init(ctx context.Context) error { // before continuing. pool.StopWait() - diff := time.Now().Sub(start) - log.WithField("duration", fmt.Sprintf("%s", diff)).Info("finished processing server configurations") + diff := time.Since(start) + log.WithField("duration", diff.String()).Info("finished processing server configurations") return nil } diff --git a/server/manager_test.go b/server/manager_test.go new file mode 100644 index 000000000..690211fd1 --- /dev/null +++ b/server/manager_test.go @@ -0,0 +1,122 @@ +package server + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/remote" +) + +// setTestConfig stores a configuration for tests that use the global one, so +// they do not depend on another test having set it first. +func setTestConfig(t *testing.T) { + t.Helper() + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + config.Set(cfg) +} + +func serverConfigurationResponse(t *testing.T, uuid string) remote.ServerConfigurationResponse { + t.Helper() + settings, err := json.Marshal(map[string]string{"uuid": uuid}) + if err != nil { + t.Fatal(err) + } + return remote.ServerConfigurationResponse{ + Settings: settings, + ProcessConfiguration: &remote.ProcessConfiguration{}, + } +} + +func TestInitServerRejectsInvalidUuid(t *testing.T) { + root := t.TempDir() + cfg, err := config.NewAtPath(filepath.Join(root, "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + cfg.System.Data = filepath.Join(root, "volumes") + config.Set(cfg) + + m := NewEmptyManager(nil) + for _, uuid := range []string{"", "..", "../other", "/srv", "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/../../other-nested"} { + t.Run(uuid, func(t *testing.T) { + if s, err := m.InitServer(uuid, serverConfigurationResponse(t, uuid)); err == nil { + t.Fatalf("expected an error for uuid %q, server root is %s", uuid, s.Filesystem().Path()) + } + }) + } + for _, p := range []string{"other", "other-nested"} { + if _, err := os.Stat(filepath.Join(root, p)); !os.IsNotExist(err) { + t.Errorf("expected %s to not be created outside of the data directory", p) + } + } +} + +func TestInitServerRejectsConfigurationForDifferentServer(t *testing.T) { + root := t.TempDir() + cfg, err := config.NewAtPath(filepath.Join(root, "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + cfg.System.Data = filepath.Join(root, "volumes") + config.Set(cfg) + + m := NewEmptyManager(nil) + requested, returned := "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + if _, err := m.InitServer(requested, serverConfigurationResponse(t, returned)); err == nil { + t.Fatal("expected an error when the Panel returns the configuration for a different server") + } + if _, err := os.Stat(filepath.Join(root, "volumes", returned)); !os.IsNotExist(err) { + t.Fatal("expected no data directory to be created for the other server") + } +} + +func TestManagerAddIfMissing(t *testing.T) { + setTestConfig(t) + m := NewEmptyManager(nil) + newServer := func() *Server { + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + if err := s.SyncWithConfiguration(serverConfigurationResponse(t, "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")); err != nil { + t.Fatal(err) + } + return s + } + + if !m.AddIfMissing(newServer()) { + t.Fatal("expected server to be added") + } + if m.AddIfMissing(newServer()) { + t.Fatal("expected a second server with the same uuid to not be added") + } + if m.Len() != 1 { + t.Fatalf("expected one server, got %d", m.Len()) + } +} + +func TestSyncWithConfigurationRejectsChangedUuid(t *testing.T) { + setTestConfig(t) + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + if err := s.SyncWithConfiguration(serverConfigurationResponse(t, "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")); err != nil { + t.Fatal(err) + } + if err := s.SyncWithConfiguration(serverConfigurationResponse(t, "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")); err == nil { + t.Fatal("expected an error when the Panel returns a different uuid for a server") + } + if s.ID() != "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" { + t.Fatalf("expected server uuid to be unchanged, got %q", s.ID()) + } +} diff --git a/server/mounts.go b/server/mounts.go index ae67f8232..2c7795cb2 100644 --- a/server/mounts.go +++ b/server/mounts.go @@ -1,9 +1,14 @@ package server import ( + "fmt" + "io/fs" + "os" "path/filepath" "strings" + "syscall" + "emperror.dev/errors" "github.com/apex/log" "github.com/Rene-Roscher/wings/config" @@ -66,7 +71,29 @@ func (s *Server) Mounts() []environment.Mount { func (s *Server) customMounts() []environment.Mount { var mounts []environment.Mount - // TODO: probably need to handle things trying to mount directories that do not exist. + // Resolving the allowed mounts touches the filesystem, which can block if one + // of them is on a network share that is down, so only do it for servers that + // have mounts. + if len(s.Config().Mounts) == 0 { + return nil + } + + cfg := config.Get() + var allowed []string + for _, a := range cfg.AllowedMounts { + resolved, err := resolveMountPath(a, cfg.System.User.Uid) + if err != nil { + s.Log().WithField("allowed_mount", a).WithField("error", err).Warn("ignoring allowed mount point") + continue + } + // Only allow the root directory if it was configured as the root directory. + if resolved == "/" && filepath.Clean(a) != "/" { + s.Log().WithField("allowed_mount", a).Warn("ignoring allowed mount point, it resolves to the root directory") + continue + } + allowed = append(allowed, resolved) + } + for _, m := range s.Config().Mounts { source := filepath.Clean(m.Source) target := filepath.Clean(m.Target) @@ -77,16 +104,22 @@ func (s *Server) customMounts() []environment.Mount { "read_only": m.ReadOnly, }) + // The source is checked against the allowed mounts once it is resolved. + resolved, err := resolveMountPath(source, cfg.System.User.Uid) + if err != nil { + logger.WithField("error", err).Warn("skipping custom server mount") + s.publishSkippedMount(target) + continue + } + mounted := false - for _, allowed := range config.Get().AllowedMounts { - // Check if the source path is included in the allowed mounts list. - // filepath.Clean will strip all trailing slashes (unless the path is a root directory). - if !strings.HasPrefix(source, filepath.Clean(allowed)) { + for _, a := range allowed { + if !isAllowedMountSource(resolved, a) { continue } mounted = true mounts = append(mounts, environment.Mount{ - Source: source, + Source: resolved, Target: target, ReadOnly: m.ReadOnly, }) @@ -95,8 +128,100 @@ func (s *Server) customMounts() []environment.Mount { if !mounted { logger.Warn("skipping custom server mount, not in list of allowed mount points") + s.publishSkippedMount(target) } } return mounts } + +// publishSkippedMount tells the user in the server console that a mount was +// skipped, since anything written to its target would otherwise silently end up +// inside the container instead. The reason is only logged, as it includes paths +// on the host. +func (s *Server) publishSkippedMount(target string) { + s.PublishConsoleOutputFromDaemon(fmt.Sprintf("Skipped the mount for %s, ask an administrator to check the Wings logs.", target)) +} + +// resolveMountPath resolves any symlinks in an absolute path that will be bind +// mounted into a container. If the path does not exist, the part of it that +// does is resolved, so that Docker will refuse to start the container as it +// always has. +// +// This returns an error if any directory above the path, before or after it is +// resolved, could be modified by the user containers run as. +func resolveMountPath(p string, containerUid int) (string, error) { + p = filepath.Clean(p) + if !filepath.IsAbs(p) { + return "", errors.New("path is not absolute") + } + + resolved, err := filepath.EvalSymlinks(p) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + // Resolve the deepest part of the path that exists. + existing, missing := p, "" + for { + existing, missing = filepath.Dir(existing), filepath.Join(filepath.Base(existing), missing) + if resolved, err = filepath.EvalSymlinks(existing); err == nil { + resolved = filepath.Join(resolved, missing) + break + } + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + } + } + + for _, v := range []string{p, resolved} { + if err := checkMountPathParents(v, containerUid); err != nil { + return "", err + } + } + return resolved, nil +} + +// mountPathCheckRoot is the highest directory checked by checkMountPathParents. +// It is only changed by tests, which cannot control the permissions of the +// directories above their temporary directory. +var mountPathCheckRoot = "/" + +// checkMountPathParents returns an error if any existing directory above the +// path is owned by the user containers run as, or is writable by its group or +// other users. +func checkMountPathParents(p string, containerUid int) error { + for d := filepath.Dir(p); ; d = filepath.Dir(d) { + st, err := os.Lstat(d) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } else if st.IsDir() { + sys, ok := st.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("unable to determine owner of %s", d) + } + if containerUid != os.Geteuid() && int(sys.Uid) == containerUid { + return fmt.Errorf("%s is owned by the user servers run as", d) + } + if st.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("%s is writable by users other than its owner", d) + } + } + if d == mountPathCheckRoot || d == "/" { + return nil + } + } +} + +// isAllowedMountSource reports whether the resolved source path is the resolved +// allowed mount point itself or a path beneath it. A plain prefix check is not +// enough since an allowed mount of "/mnt" would otherwise also allow "/mnt2". +func isAllowedMountSource(source string, allowed string) bool { + if source == allowed || allowed == string(filepath.Separator) { + return true + } + return strings.HasPrefix(source, allowed+string(filepath.Separator)) +} diff --git a/server/mounts_test.go b/server/mounts_test.go new file mode 100644 index 000000000..dc2c100ca --- /dev/null +++ b/server/mounts_test.go @@ -0,0 +1,258 @@ +package server + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/apex/log" + "github.com/apex/log/handlers/memory" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/events" +) + +// setupMountTest creates a directory to hold mount sources and points the +// configuration at it. The permissions of the directories above it are outside +// the control of the test, so they are not checked. +func setupMountTest(t *testing.T, allowed ...string) string { + t.Helper() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := os.Chmod(root, 0o755); err != nil { + t.Fatal(err) + } + previous := mountPathCheckRoot + mountPathCheckRoot = root + t.Cleanup(func() { mountPathCheckRoot = previous }) + + cfg, err := config.NewAtPath(filepath.Join(t.TempDir(), "config.yml")) + if err != nil { + t.Fatal(err) + } + cfg.AuthenticationToken = "test-token" + for _, a := range allowed { + cfg.AllowedMounts = append(cfg.AllowedMounts, filepath.Join(root, a)) + } + config.Set(cfg) + return root +} + +func mkdirs(t *testing.T, root string, dirs ...string) { + t.Helper() + for _, d := range dirs { + if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil { + t.Fatal(err) + } + } +} + +func symlink(t *testing.T, target string, link string) { + t.Helper() + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } +} + +// mountSource returns the source of the custom mount for the given path, or an +// empty string if it was not mounted. +func mountSource(t *testing.T, source string) string { + t.Helper() + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + s.cfg.Mounts = []Mount{{Source: source, Target: "/data"}} + mounts := s.customMounts() + if len(mounts) == 0 { + return "" + } + return mounts[0].Source +} + +func TestCustomMountsRequirePathContainment(t *testing.T) { + root := setupMountTest(t, "mnt/") + mkdirs(t, root, "mnt/data", "mnt2/data", "mntfoo", "outside") + // A symlink inside of the allowed mount that points outside of it. + symlink(t, filepath.Join(root, "outside"), filepath.Join(root, "mnt", "link")) + + tests := []struct { + source string + allowed bool + }{ + {source: "mnt", allowed: true}, + {source: "mnt/data", allowed: true}, + {source: "mnt/data/../data", allowed: true}, + {source: "mnt2/data", allowed: false}, + {source: "mntfoo", allowed: false}, + {source: "mnt/../outside", allowed: false}, + {source: "mnt/link", allowed: false}, + {source: "mnt/link/missing", allowed: false}, + } + + for _, tt := range tests { + t.Run(tt.source, func(t *testing.T) { + got := mountSource(t, filepath.Join(root, tt.source)) + if tt.allowed && got == "" { + t.Fatalf("expected %s to be mounted", tt.source) + } + if !tt.allowed && got != "" { + t.Fatalf("expected %s to be rejected, got %s", tt.source, got) + } + }) + } +} + +// A source that does not exist is still passed along so that Docker refuses to +// start the server, rather than it starting without the mount. +func TestCustomMountsKeepMissingSources(t *testing.T) { + root := setupMountTest(t, "mnt") + mkdirs(t, root, "mnt") + + want := filepath.Join(root, "mnt", "missing", "data") + if got := mountSource(t, want); got != want { + t.Fatalf("expected missing source to be mounted as %s, got %q", want, got) + } +} + +// Sources beneath a directory that is writable by other users, or owned by the +// server user, are not mounted. +func TestCustomMountsRequireProtectedParentDirectories(t *testing.T) { + for _, mode := range []os.FileMode{0o777, 0o775, 0o757} { + t.Run(mode.String(), func(t *testing.T) { + root := setupMountTest(t, "mnt") + mkdirs(t, root, "mnt/shared/maps") + if err := os.Chmod(filepath.Join(root, "mnt", "shared"), mode); err != nil { + t.Fatal(err) + } + + if got := mountSource(t, filepath.Join(root, "mnt", "shared", "maps")); got != "" { + t.Fatalf("expected source beneath a writable directory to be rejected, got %s", got) + } + // The writable directory itself can still be mounted. + if got := mountSource(t, filepath.Join(root, "mnt", "shared")); got == "" { + t.Fatal("expected the writable directory itself to be mounted") + } + }) + } + + t.Run("owned by the server user", func(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("changing the owner of a directory requires root") + } + root := setupMountTest(t, "mnt") + mkdirs(t, root, "mnt/shared/maps") + if err := os.Chown(filepath.Join(root, "mnt", "shared"), 12345, 12345); err != nil { + t.Fatal(err) + } + config.Update(func(c *config.Configuration) { + c.System.User.Uid = 12345 + c.System.User.Gid = 12345 + }) + + if got := mountSource(t, filepath.Join(root, "mnt", "shared", "maps")); got != "" { + t.Fatalf("expected source beneath a directory owned by the server user to be rejected, got %s", got) + } + }) +} + +func TestCustomMountsResolveAllowedMounts(t *testing.T) { + t.Run("symlinked allowed mount", func(t *testing.T) { + root := setupMountTest(t, "data") + mkdirs(t, root, "disk1/maps") + symlink(t, filepath.Join(root, "disk1"), filepath.Join(root, "data")) + + want := filepath.Join(root, "disk1", "maps") + if got := mountSource(t, filepath.Join(root, "data", "maps")); got != want { + t.Fatalf("expected source to be mounted as %s, got %q", want, got) + } + }) + + t.Run("allowed mount resolving to the root directory", func(t *testing.T) { + root := setupMountTest(t, "shared") + symlink(t, "/", filepath.Join(root, "shared")) + + if got := mountSource(t, "/usr"); got != "" { + t.Fatalf("expected an allowed mount resolving to / to be ignored, got %s", got) + } + }) + + t.Run("allowed mount beneath a writable directory", func(t *testing.T) { + root := setupMountTest(t, "volumes/server/shared") + mkdirs(t, root, "volumes/server", "other") + symlink(t, filepath.Join(root, "other"), filepath.Join(root, "volumes", "server", "shared")) + if err := os.Chmod(filepath.Join(root, "volumes", "server"), 0o777); err != nil { + t.Fatal(err) + } + + if got := mountSource(t, filepath.Join(root, "other")); got != "" { + t.Fatalf("expected an allowed mount beneath a writable directory to be ignored, got %s", got) + } + }) +} + +// Resolving the allowed mounts touches the filesystem, which can block on a +// network share that is down, so servers without mounts should not do it. +func TestCustomMountsWithoutMountsDoNotResolveAllowedMounts(t *testing.T) { + setupMountTest(t) + config.Update(func(c *config.Configuration) { + c.AllowedMounts = []string{"not/absolute"} + }) + handler := memory.New() + logger := log.Log.(*log.Logger) + previous := logger.Handler + logger.Handler = handler + t.Cleanup(func() { logger.Handler = previous }) + + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + if mounts := s.customMounts(); len(mounts) != 0 { + t.Fatalf("expected no custom mounts, got %v", mounts) + } + for _, e := range handler.Entries { + if strings.Contains(e.Message, "allowed mount") { + t.Fatalf("expected allowed mounts to not be resolved, got log entry %q", e.Message) + } + } +} + +// A skipped mount is otherwise only visible in the Wings logs, while anything +// written to its target silently ends up inside the container. +func TestSkippedCustomMountIsReportedInConsole(t *testing.T) { + root := setupMountTest(t, "mnt") + mkdirs(t, root, "mnt", "outside") + + s, err := New(nil) + if err != nil { + t.Fatal(err) + } + ch := make(chan []byte, 8) + s.Events().On(ch) + s.cfg.Mounts = []Mount{{Source: filepath.Join(root, "outside"), Target: "/data"}} + if mounts := s.customMounts(); len(mounts) != 0 { + t.Fatalf("expected the mount to be skipped, got %v", mounts) + } + + select { + case b := <-ch: + var e events.Event + if err := events.DecodeTo(b, &e); err != nil { + t.Fatal(err) + } + msg, _ := e.Data.(string) + if e.Topic != ConsoleOutputEvent || !strings.Contains(msg, "/data") { + t.Fatalf("expected a console message about the /data mount, got %s %q", e.Topic, msg) + } + if strings.Contains(msg, root) { + t.Fatalf("expected the console message to not include host paths, got %q", msg) + } + case <-time.After(time.Second * 5): + t.Fatal("expected a console message about the skipped mount") + } +} diff --git a/server/power.go b/server/power.go index 75fbc3f49..155f09e07 100644 --- a/server/power.go +++ b/server/power.go @@ -10,6 +10,7 @@ import ( "github.com/Rene-Roscher/wings/config" "github.com/Rene-Roscher/wings/environment" + "github.com/Rene-Roscher/wings/system" ) type PowerAction string @@ -117,6 +118,9 @@ func (s *Server) HandlePowerAction(action PowerAction, waitSeconds ...int) error if err := s.powerLock.Acquire(); err == nil { log.Info("acquired exclusive lock on power actions, processing event...") defer cleanup() + } else if errors.Is(err, system.ErrLockerDestroyed) { + // The server is being deleted. + return err } else { log.Warn("failed to acquire exclusive lock, ignoring failure for termination event") } @@ -171,6 +175,8 @@ func (s *Server) HandlePowerAction(action PowerAction, waitSeconds ...int) error // Execute a few functions before actually calling the environment start commands. This ensures // that everything is ready to go for environment booting, and that the server can even be started. func (s *Server) onBeforeStart() error { + // Sync also applies the new environment variables and process resource limits + // to the environment. s.Log().Info("syncing server configuration with panel") if err := s.Sync(); err != nil { return errors.WithMessage(err, "unable to sync server data from Panel instance") @@ -182,10 +188,6 @@ func (s *Server) onBeforeStart() error { return ErrSuspended } - // Ensure we sync the server information with the environment so that any new environment variables - // and process resource limits are correctly applied. - s.SyncWithEnvironment() - // If a server has unlimited disk space, we don't care enough to block the startup to check remaining. // However, we should trigger a size anyway, as it'd be good to kick it off for other processes. if s.DiskSpace() <= 0 { diff --git a/server/power_destroy_test.go b/server/power_destroy_test.go new file mode 100644 index 000000000..ec68125ba --- /dev/null +++ b/server/power_destroy_test.go @@ -0,0 +1,30 @@ +package server + +import ( + "testing" + + "emperror.dev/errors" + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/system" +) + +// Power actions that arrive while a server is being deleted return an error. +func TestPowerActionAfterCleanupForDestroyReturnsError(t *testing.T) { + s, err := New(nil) + require.NoError(t, err) + + s.CleanupForDestroy() + + for _, action := range []PowerAction{PowerActionStart, PowerActionStop, PowerActionRestart, PowerActionTerminate} { + var err error + require.NotPanics(t, func() { + err = s.HandlePowerAction(action) + }, "action %s", action) + require.True(t, errors.Is(err, system.ErrLockerDestroyed), "action %s returned %v", action, err) + } + + // A second cleanup, from a repeated delete request, is a no-op. + require.NotPanics(t, s.CleanupForDestroy) + require.False(t, s.ExecutingPowerAction()) +} diff --git a/server/power_test.go b/server/power_test.go index 2a076bbd0..ff72f4c18 100644 --- a/server/power_test.go +++ b/server/power_test.go @@ -16,7 +16,7 @@ func TestPower(t *testing.T) { s := &Server{powerLock: system.NewLocker()} g.Assert(s.ExecutingPowerAction()).IsFalse() - s.powerLock.Acquire() + g.Assert(s.powerLock.Acquire()).IsNil() g.Assert(s.ExecutingPowerAction()).IsTrue() }) }) diff --git a/server/resources.go b/server/resources.go index 64118853e..11396b902 100644 --- a/server/resources.go +++ b/server/resources.go @@ -34,8 +34,11 @@ func (s *Server) Proc() ResourceUsage { defer s.resources.mu.Unlock() // Store the updated disk usage when requesting process usage. atomic.StoreInt64(&s.resources.Disk, s.Filesystem().CachedUsage()) - //goland:noinspection GoVetCopyLock - return s.resources + return ResourceUsage{ + Stats: s.resources.Stats, + State: s.resources.State, + Disk: s.resources.Disk, + } } // UpdateStats updates the current stats for the server's resource usage. diff --git a/server/server.go b/server/server.go index 57b019a55..f3f1f5309 100644 --- a/server/server.go +++ b/server/server.go @@ -13,6 +13,7 @@ import ( "emperror.dev/errors" "github.com/apex/log" + "github.com/asaskevich/govalidator" "github.com/creasty/defaults" "github.com/Rene-Roscher/wings/config" @@ -75,17 +76,14 @@ type Server struct { sftpBag *system.ContextBag sinks map[system.SinkName]*system.SinkPool - - logSink *system.SinkPool - installSink *system.SinkPool } // AtomicStateTransition represents an atomic change to server operational state type AtomicStateTransition struct { EnvironmentState string - BackingUp *bool // nil = no change - Restoring *bool // nil = no change - Transferring *bool // nil = no change + BackingUp *bool // nil = no change + Restoring *bool // nil = no change + Transferring *bool // nil = no change } // ApplyAtomicStateTransition atomically applies state changes to prevent race conditions @@ -96,7 +94,7 @@ func (s *Server) ApplyAtomicStateTransition(transition AtomicStateTransition) { "before_transferring": s.transferring.Load(), "before_env_state": s.Environment.State(), }).Debug("ATOMIC STATE TRANSITION: before") - + // Apply all atomic state changes together if transition.BackingUp != nil { s.Log().WithField("new_backing_up", *transition.BackingUp).Debug("setting BackingUp flag") @@ -110,13 +108,13 @@ func (s *Server) ApplyAtomicStateTransition(transition AtomicStateTransition) { s.Log().WithField("new_transferring", *transition.Transferring).Debug("setting Transferring flag") s.transferring.Store(*transition.Transferring) } - + // Finally set environment state if transition.EnvironmentState != "" { s.Log().WithField("new_env_state", transition.EnvironmentState).Debug("setting Environment state") s.Environment.SetState(transition.EnvironmentState) } - + s.Log().WithFields(log.Fields{ "after_backing_up": s.backingUp.Load(), "after_restoring": s.restoring.Load(), @@ -162,24 +160,16 @@ func (s *Server) CleanupForDestroy() { s.DestroyAllSinks() s.Websockets().CancelAll() s.powerLock.Destroy() - - // CRITICAL: Cancel all running backup operations for this server - // This prevents resource leaks and inconsistent states during server deletion - registry := GetBackupOperationRegistry() - if err := registry.CancelAllForServer(s.ID()); err != nil { - s.Log().WithError(err).Error("failed to cancel backup operations during server cleanup") +} + +// CancelBackupOperations cancels every running or queued backup and restore of +// this server. It is keyed by UUID, so it must only be called when the server +// itself is deleted, never for a throwaway duplicate instance that shares the +// UUID of a live server (duplicate create, transfer cleanup). +func (s *Server) CancelBackupOperations() { + if err := GetBackupOperationRegistry().CancelAllForServer(s.ID()); err != nil { + s.Log().WithError(err).Error("failed to cancel backup operations during server deletion") } - - // CRITICAL: Clean up local backup files to prevent disk space leaks - // This removes orphaned backup files when the server is deleted - go func() { - // Run backup file cleanup in background to avoid blocking server deletion - // Import is required here since backup package imports server package (circular import) - // We'll call this via a method on the server instead - if err := s.cleanupBackupFiles(); err != nil { - s.Log().WithError(err).Error("failed to clean up backup files during server deletion") - } - }() } // ID returns the UUID for the server instance. @@ -287,24 +277,28 @@ func (s *Server) Sync() error { // can be called from scoped where the server may not be fully initialized, // therefore other things like the filesystem and environment may not exist yet. func (s *Server) SyncWithConfiguration(cfg remote.ServerConfigurationResponse) error { - c := Configuration{ + c := configurationData{ CrashDetectionEnabled: config.Get().System.CrashDetection.CrashDetectionEnabled, } if err := json.Unmarshal(cfg.Settings, &c); err != nil { return errors.WithStackIf(err) } + // Only accept a UUID from the Panel, and never allow it to change once the + // server has been created. + if !govalidator.IsUUID(c.Uuid) { + return errors.New("server: configuration returned by the Panel contains an invalid server uuid") + } + if id := s.ID(); id != "" && id != c.Uuid { + return errors.New("server: configuration returned by the Panel is for a different server") + } + s.cfg.mu.Lock() defer s.cfg.mu.Unlock() - // Lock the new configuration. Since we have the deferred Unlock above we need - // to make sure that the NEW configuration object is already locked since that - // defer is running on the memory address for "s.cfg.mu" which we're explicitly - // changing on the next line. - c.mu.Lock() - - //goland:noinspection GoVetCopyLock - s.cfg = c + // Only replace the values, never the mutex, since other goroutines may be + // waiting to acquire it and would never be woken if it were overwritten. + s.cfg.configurationData = c s.Lock() s.procConfig = cfg.ProcessConfiguration @@ -333,7 +327,7 @@ func (s *Server) CreateEnvironment() error { // without any dashes. p := filepath.Join(cfg.System.MachineID.Directory, s.ID()) machineID := append(bytes.ReplaceAll([]byte(s.ID()), []byte{'-'}, []byte{}), '\n') - if err := os.WriteFile(p, machineID, 0o644); err != nil { + if err := os.WriteFile(p, machineID, 0o644); err != nil { //nolint:gosec // mounted as /etc/machine-id, which the container user must be able to read return fmt.Errorf("failed to write machine-id (at '%s') for server '%s': %w", p, s.ID(), err) } } @@ -449,7 +443,7 @@ func (s *Server) ToAPIResponse() APIResponse { State: s.Environment.State(), IsSuspended: s.IsSuspended(), Utilization: s.Proc(), - Configuration: *s.Config(), + Configuration: s.cfg.snapshot(), } } @@ -457,99 +451,3 @@ func (s *Server) ToAPIResponse() APIResponse { func (s *Server) PublishActivity(event string, data map[string]any) { s.Events().Publish(ActivityEvent, data) } - -// cleanupBackupFiles removes all local backup files associated with this server -// This method is called during server deletion to prevent orphaned backup files -func (s *Server) cleanupBackupFiles() error { - // We need to avoid circular imports since backup package imports server package - // For now, we'll implement a basic cleanup using the same logic as in backup_local.go - // but without importing the backup package - - backupDir := config.Get().System.BackupDirectory - logger := s.Log().WithFields(log.Fields{ - "server_id": s.ID(), - "backup_dir": backupDir, - }) - - // List all files in backup directory - files, err := os.ReadDir(backupDir) - if err != nil { - if os.IsNotExist(err) { - logger.Debug("backup directory does not exist, nothing to clean up") - return nil - } - return errors.WrapIf(err, "failed to read backup directory") - } - - var removedFiles []string - var failedRemovals []string - - // Common backup file extensions - backupExtensions := []string{".tar.gz", ".tar", ".gz"} - - // Iterate through all files and find backup files - for _, file := range files { - if file.IsDir() { - continue - } - - fileName := file.Name() - - // Check if this file looks like a backup file - isBackupFile := false - lowerName := strings.ToLower(fileName) - for _, ext := range backupExtensions { - if strings.HasSuffix(lowerName, ext) { - isBackupFile = true - break - } - } - - if !isBackupFile { - continue - } - - // Extract potential backup UUID from filename (before first dot) - parts := strings.Split(fileName, ".") - if len(parts) < 2 { - continue // Not a valid backup file format - } - - backupUUID := parts[0] - - // Skip if the UUID doesn't look valid (should be 36 characters for UUID) - if len(backupUUID) != 36 { - continue - } - - filePath := filepath.Join(backupDir, fileName) - - logger.WithField("file", fileName).Debug("found backup file, attempting removal") - - if err := os.Remove(filePath); err != nil { - logger.WithError(err).WithField("file", fileName).Error("failed to remove backup file") - failedRemovals = append(failedRemovals, fileName) - } else { - logger.WithField("file", fileName).Info("removed backup file") - removedFiles = append(removedFiles, fileName) - } - } - - // Log summary - if len(removedFiles) > 0 { - logger.WithFields(log.Fields{ - "removed_count": len(removedFiles), - "removed_files": removedFiles, - }).Info("cleaned up backup files for server") - } - - if len(failedRemovals) > 0 { - logger.WithFields(log.Fields{ - "failed_count": len(failedRemovals), - "failed_files": failedRemovals, - }).Warn("some backup files could not be removed") - return errors.New("failed to remove some backup files") - } - - return nil -} diff --git a/server/transfer/manager.go b/server/transfer/manager.go index 9883b5cf6..e54683e3f 100644 --- a/server/transfer/manager.go +++ b/server/transfer/manager.go @@ -32,20 +32,28 @@ func NewManager() *Manager { } } -// Add adds a transfer to the manager. -func (m *Manager) Add(transfer *Transfer) { +// Add adds a transfer to the manager, returning false if there is already a +// transfer for the server. +func (m *Manager) Add(transfer *Transfer) bool { m.mu.Lock() defer m.mu.Unlock() + if _, ok := m.transfers[transfer.Server.ID()]; ok { + return false + } m.transfers[transfer.Server.ID()] = transfer + return true } -// Remove removes a transfer from the manager. +// Remove removes a transfer from the manager. A different transfer for the same +// server is left in place. func (m *Manager) Remove(transfer *Transfer) { m.mu.Lock() defer m.mu.Unlock() - delete(m.transfers, transfer.Server.ID()) + if m.transfers[transfer.Server.ID()] == transfer { + delete(m.transfers, transfer.Server.ID()) + } } // Get gets a transfer from the manager using a server ID. diff --git a/server/transfer/manager_test.go b/server/transfer/manager_test.go new file mode 100644 index 000000000..8809734f9 --- /dev/null +++ b/server/transfer/manager_test.go @@ -0,0 +1,35 @@ +package transfer + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Rene-Roscher/wings/server" +) + +func newManagerTestTransfer(t *testing.T, uuid string) *Transfer { + t.Helper() + s, err := server.New(nil) + require.NoError(t, err) + s.Config().Uuid = uuid + return New(context.Background(), s) +} + +// Only one transfer can be tracked for a server, and removing one transfer does +// not remove a different transfer for the same server. +func TestManagerTracksOneTransferPerServer(t *testing.T) { + m := NewManager() + first := newManagerTestTransfer(t, "server") + second := newManagerTestTransfer(t, "server") + + require.True(t, m.Add(first)) + require.False(t, m.Add(second)) + + m.Remove(second) + require.Same(t, first, m.Get("server")) + + m.Remove(first) + require.Nil(t, m.Get("server")) +} diff --git a/server/websockets.go b/server/websockets.go index 6aa034351..c6823ad0e 100644 --- a/server/websockets.go +++ b/server/websockets.go @@ -9,7 +9,13 @@ import ( type WebsocketBag struct { mu sync.Mutex - conns map[uuid.UUID]*context.CancelFunc + conns map[uuid.UUID]websocketConn +} + +type websocketConn struct { + cancel *context.CancelFunc + // user returns the user the connection is currently authenticated as. + user func() string } // Websockets returns the websocket bag which contains all the currently open websocket connections @@ -38,10 +44,28 @@ func (w *WebsocketBag) Push(u uuid.UUID, cancel *context.CancelFunc) { defer w.mu.Unlock() if w.conns == nil { - w.conns = make(map[uuid.UUID]*context.CancelFunc) + w.conns = make(map[uuid.UUID]websocketConn) + } + + w.conns[u] = websocketConn{cancel: cancel} +} + +// TryPush adds a new websocket connection to the stack if there are fewer than +// max connections in it, returning false if the connection was not added. user +// returns the user the connection is currently authenticated as. +func (w *WebsocketBag) TryPush(u uuid.UUID, cancel *context.CancelFunc, user func() string, max int) bool { + w.mu.Lock() + defer w.mu.Unlock() + + if w.conns == nil { + w.conns = make(map[uuid.UUID]websocketConn) + } + if _, ok := w.conns[u]; !ok && len(w.conns) >= max { + return false } - w.conns[u] = cancel + w.conns[u] = websocketConn{cancel: cancel, user: user} + return true } // Remove removes a connection from the stack. @@ -58,11 +82,24 @@ func (w *WebsocketBag) CancelAll() { defer w.mu.Unlock() if w.conns != nil { - for _, cancel := range w.conns { - (*cancel)() + for _, c := range w.conns { + (*c.cancel)() } } // Reset the connections. - w.conns = make(map[uuid.UUID]*context.CancelFunc) + w.conns = make(map[uuid.UUID]websocketConn) +} + +// CancelUser disconnects every websocket that is authenticated as the user. +func (w *WebsocketBag) CancelUser(user string) { + w.mu.Lock() + defer w.mu.Unlock() + + for u, c := range w.conns { + if c.user != nil && c.user() == user { + (*c.cancel)() + delete(w.conns, u) + } + } } diff --git a/sftp/event.go b/sftp/event.go index 4c8795ae8..8add180b9 100644 --- a/sftp/event.go +++ b/sftp/event.go @@ -59,7 +59,7 @@ func (eh *eventHandler) Log(e models.Event, fa FileAction) error { go func() { defer func() { if r := recover(); r != nil { - // Cannot access server logger here, so no logging + log.WithField("panic", r).WithField("event", e).Warn("sftp: activity publisher panicked") } }() eh.publisher.PublishActivity("activity", map[string]any{ diff --git a/sftp/handler.go b/sftp/handler.go index 7e52d7137..6760430dd 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "emperror.dev/errors" "github.com/apex/log" @@ -24,6 +25,8 @@ const ( PermissionFileUpdate = "file.update" PermissionFileDelete = "file.delete" sftpAttributeExtended = 1 << 31 + // maxOpenFiles is the number of files a session may have open at once. + maxOpenFiles = 64 ) type Handler struct { @@ -34,11 +37,49 @@ type Handler struct { permissions []string logger *log.Entry ro bool + open atomic.Int32 +} + +// openFile returns a slot for a file opened by the session, or nil if the +// session already has the most files open that it may. +func (h *Handler) openFile() *openFile { + if h.open.Add(1) > maxOpenFiles { + h.open.Add(-1) + return nil + } + return &openFile{release: func() { h.open.Add(-1) }} +} + +// openFile is a session's slot for an open file, released when it is closed. +type openFile struct { + once sync.Once + release func() +} + +func (f *openFile) close() { + if f != nil { + f.once.Do(f.release) + } +} + +// readerAt is a file opened for reading by a session. +type readerAt struct { + io.ReaderAt + slot *openFile +} + +func (r readerAt) Close() error { + r.slot.close() + if c, ok := r.ReaderAt.(io.Closer); ok { + return c.Close() + } + return nil } type quotaWriterAt struct { io.WriterAt server *server.Server + slot *openFile } func (w quotaWriterAt) WriteAt(p []byte, off int64) (int, error) { @@ -54,6 +95,7 @@ func (w quotaWriterAt) WriteAt(p []byte, off int64) (int, error) { } func (w quotaWriterAt) Close() error { + w.slot.close() if c, ok := w.WriterAt.(io.Closer); ok { return c.Close() } @@ -103,17 +145,22 @@ func (h *Handler) Fileread(request *sftp.Request) (io.ReaderAt, error) { if !h.can(PermissionFileReadContent) { return nil, sftp.ErrSSHFxPermissionDenied } + slot := h.openFile() + if slot == nil { + return nil, sftp.ErrSSHFxFailure + } h.mu.Lock() defer h.mu.Unlock() f, _, err := h.fs.File(request.Filepath) if err != nil { + slot.close() if !errors.Is(err, os.ErrNotExist) { h.logger.WithField("error", err).Error("error processing readfile request") return nil, sftp.ErrSSHFxFailure } return nil, sftp.ErrSSHFxNoSuchFile } - return f, nil + return readerAt{ReaderAt: f, slot: slot}, nil } // Filewrite handles the write actions for a file on the system. @@ -147,8 +194,13 @@ func (h *Handler) Filewrite(request *sftp.Request) (io.WriterAt, error) { if !h.can(permission) { return nil, sftp.ErrSSHFxPermissionDenied } + slot := h.openFile() + if slot == nil { + return nil, sftp.ErrSSHFxFailure + } f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_TRUNC) if err != nil { + slot.close() l.WithField("flags", request.Flags).WithField("error", err).Error("failed to open existing file on system") return nil, sftp.ErrSSHFxFailure } @@ -160,7 +212,7 @@ func (h *Handler) Filewrite(request *sftp.Request) (io.WriterAt, error) { event = server.ActivitySftpCreate } h.events.MustLog(event, FileAction{Entity: request.Filepath}) - return quotaWriterAt{WriterAt: f, server: h.server}, nil + return quotaWriterAt{WriterAt: f, server: h.server, slot: slot}, nil } func setstatMode(request *sftp.Request) (os.FileMode, error) { @@ -215,7 +267,6 @@ func (h *Handler) Filecmd(request *sftp.Request) error { l.WithField("error", err).Error("failed to perform setstat on item") return sftp.ErrSSHFxFailure } - break // Support renaming a file (aka Move). case "Rename": if !h.can(PermissionFileUpdate) { @@ -229,7 +280,6 @@ func (h *Handler) Filecmd(request *sftp.Request) error { return sftp.ErrSSHFxFailure } h.events.MustLog(server.ActivitySftpRename, FileAction{Entity: request.Filepath, Target: request.Target}) - break // Handle deletion of a directory. This will properly delete all of the files and // folders within that directory if it is not already empty (unlike a lot of SFTP // clients that must delete each file individually). @@ -256,7 +306,6 @@ func (h *Handler) Filecmd(request *sftp.Request) error { return sftp.ErrSSHFxFailure } h.events.MustLog(server.ActivitySftpCreateDirectory, FileAction{Entity: request.Filepath}) - break // Support creating symlinks between files. The source and target must resolve within // the server home directory. case "Symlink": @@ -267,7 +316,6 @@ func (h *Handler) Filecmd(request *sftp.Request) error { l.WithField("target", request.Target).WithField("error", err).Error("failed to create symlink") return sftp.ErrSSHFxFailure } - break // Called when deleting a file. case "Remove": if !h.can(PermissionFileDelete) { diff --git a/sftp/limits_test.go b/sftp/limits_test.go new file mode 100644 index 000000000..d98af558c --- /dev/null +++ b/sftp/limits_test.go @@ -0,0 +1,26 @@ +package sftp + +import ( + "testing" + + "github.com/Rene-Roscher/wings/config" +) + +func TestConfiguredLimits(t *testing.T) { + if got := configuredLimits(config.SftpConfiguration{}); got != defaultLimits { + t.Fatalf("expected the default limits when none are configured, got %+v", got) + } + + got := configuredLimits(config.SftpConfiguration{MaxConnectionsPerUser: 64, MaxAuthenticatingConnectionsPerIP: 200}) + if got.maxPerUser != 64 || got.maxPendingPerIP != 200 { + t.Fatalf("expected the configured limits to be used, got %+v", got) + } + // All connections may come from a single proxy, so the limit for all addresses + // cannot be lower than the limit for one. + if got.maxPending != 200 { + t.Fatalf("expected the limit for all addresses to allow the limit for one, got %d", got.maxPending) + } + if got.handshakeTimeout != defaultLimits.handshakeTimeout { + t.Fatalf("expected the default handshake timeout, got %s", got.handshakeTimeout) + } +} diff --git a/sftp/server.go b/sftp/server.go index 7209161da..0cfc13004 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -7,11 +7,14 @@ import ( "encoding/pem" "io" "net" + "net/netip" "os" "path" "regexp" "strconv" "strings" + "sync" + "time" "emperror.dev/errors" "github.com/apex/log" @@ -20,6 +23,7 @@ import ( "golang.org/x/crypto/ssh" "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/internal/network" "github.com/Rene-Roscher/wings/remote" "github.com/Rene-Roscher/wings/server" ) @@ -29,12 +33,125 @@ import ( // server and sending a flood of usernames. var validUsernameRegexp = regexp.MustCompile(`^(?i)(.+)\.([a-z0-9]{8})$`) +// connectionLimits are the limits applied to connections. +type connectionLimits struct { + // handshakeTimeout is the amount of time a connection has to complete the + // handshake and authenticate. + handshakeTimeout time.Duration + // maxPending is the maximum number of connections that may be handshaking + // at once. + maxPending int + // maxPendingPerIP is the maximum number of connections from a single IP + // address, or IPv6 /64 network, that may be handshaking at once. + maxPendingPerIP int + // maxPerUser is the maximum number of authenticated connections a single + // user may have open at once. + maxPerUser int +} + +// defaultLimits are the connection limits used by the SFTP server. The time +// allowed to authenticate matches the default LoginGraceTime of OpenSSH. +var defaultLimits = connectionLimits{ + handshakeTimeout: 2 * time.Minute, + maxPending: 128, + maxPendingPerIP: 16, + maxPerUser: 32, +} + //goland:noinspection GoNameStartsWithPackageName type SFTPServer struct { manager *server.Manager BasePath string ReadOnly bool Listen string + // MaxConnections is the maximum number of connections that may be open at + // once, or zero for no limit. + MaxConnections int + + limits connectionLimits + pending connectionCounter + sessions connectionCounter + // Dropped connections are logged separately for each reason, so that one + // reason does not hide the other. + droppedPending droppedLog + droppedUser droppedLog +} + +// connectionCounter counts open connections in total and by a key, such as the +// address or the user they belong to. +type connectionCounter struct { + mu sync.Mutex + total int + byKey map[string]int +} + +// acquire counts a new connection for the key, returning false if that would +// exceed either limit. A limit of zero is not enforced. +func (cc *connectionCounter) acquire(key string, maxTotal int, maxPerKey int) bool { + cc.mu.Lock() + defer cc.mu.Unlock() + if cc.byKey == nil { + cc.byKey = make(map[string]int) + } + if (maxTotal > 0 && cc.total >= maxTotal) || (maxPerKey > 0 && cc.byKey[key] >= maxPerKey) { + return false + } + cc.total++ + cc.byKey[key]++ + return true +} + +func (cc *connectionCounter) release(key string) { + cc.mu.Lock() + defer cc.mu.Unlock() + cc.total-- + if cc.byKey[key] <= 1 { + delete(cc.byKey, key) + } else { + cc.byKey[key]-- + } +} + +// droppedLog limits how often dropped connections are logged. +type droppedLog struct { + mu sync.Mutex + last time.Time + count int +} + +func (d *droppedLog) log(message string, key string) { + d.mu.Lock() + d.count++ + if time.Since(d.last) < time.Second*10 { + d.mu.Unlock() + return + } + count := d.count + d.count = 0 + d.last = time.Now() + d.mu.Unlock() + log.WithField("key", key).WithField("dropped", count).Warn("sftp: " + message) +} + +// connectionKey returns the key used to limit connections from the address. All +// of the addresses in an IPv6 /64 network belong to the same user, so they share +// a key. +func connectionKey(addr net.Addr) string { + host, _, err := net.SplitHostPort(addr.String()) + if err != nil { + return addr.String() + } + ip, err := netip.ParseAddr(host) + if err != nil { + return host + } + ip = ip.Unmap().WithZone("") + if ip.Is6() { + if p, err := ip.Prefix(64); err == nil { + return p.String() + } + } + return ip.String() } func New(m *server.Manager) *SFTPServer { @@ -44,9 +161,26 @@ func New(m *server.Manager) *SFTPServer { BasePath: cfg.Data, ReadOnly: cfg.Sftp.ReadOnly, Listen: cfg.Sftp.Address + ":" + strconv.Itoa(cfg.Sftp.Port), + limits: configuredLimits(cfg.Sftp), } } +// configuredLimits returns the default connection limits with any overrides +// from the configuration applied. +func configuredLimits(cfg config.SftpConfiguration) connectionLimits { + limits := defaultLimits + if cfg.MaxConnectionsPerUser > 0 { + limits.maxPerUser = cfg.MaxConnectionsPerUser + } + if cfg.MaxAuthenticatingConnectionsPerIP > 0 { + limits.maxPendingPerIP = cfg.MaxAuthenticatingConnectionsPerIP + // Every connection may come from the address of a proxy, so the limit for + // all addresses must allow at least as many. + limits.maxPending = max(limits.maxPending, limits.maxPendingPerIP) + } + return limits +} + // Run starts the SFTP server and add a persistent listener to handle inbound // SFTP connections. This will automatically generate an ED25519 key if one does // not already exist on the system for host key verification purposes. @@ -98,30 +232,111 @@ func (c *SFTPServer) Run() error { if err != nil { return err } + if c.MaxConnections > 0 { + listener = network.LimitListener(listener, c.MaxConnections, "sftp") + } public := string(ssh.MarshalAuthorizedKey(private.PublicKey())) log.WithField("listen", c.Listen).WithField("public_key", strings.Trim(public, "\n")).Info("sftp server listening for connections") + return c.serve(listener, conf) +} + +// serve accepts connections on the listener until it is closed. +func (c *SFTPServer) serve(listener net.Listener, conf *ssh.ServerConfig) error { + var delay time.Duration for { - if conn, _ := listener.Accept(); conn != nil { - go func(conn net.Conn) { - defer conn.Close() - if err := c.AcceptInbound(conn, conf); err != nil { - log.WithField("error", err).WithField("ip", conn.RemoteAddr().String()).Error("sftp: failed to accept inbound connection") - } - }(conn) + conn, err := listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return err + } + // Accept fails immediately while the process is out of file descriptors, + // so back off rather than spinning on it. + if delay == 0 { + delay = 5 * time.Millisecond + } else { + delay *= 2 + } + if delay > time.Second { + delay = time.Second + } + log.WithField("error", err).WithField("retry_in", delay).Error("sftp: failed to accept connection") + time.Sleep(delay) + continue } + delay = 0 + + key := connectionKey(conn.RemoteAddr()) + if !c.pending.acquire(key, c.limits.maxPending, c.limits.maxPendingPerIP) { + c.droppedPending.log("too many unauthenticated connections, dropping connections", key) + _ = conn.Close() + continue + } + + go func(conn net.Conn, key string) { + defer conn.Close() + sconn, chans, reqs, err := handshake(conn, conf, c.limits.handshakeTimeout) + c.pending.release(key) + if err != nil { + log.WithField("error", err).WithField("ip", conn.RemoteAddr().String()).Error("sftp: failed to accept inbound connection") + return + } + user := sessionKey(sconn) + if !c.sessions.acquire(user, 0, c.limits.maxPerUser) { + c.droppedUser.log("too many connections for user, dropping connections", user) + _ = sconn.Close() + return + } + defer c.sessions.release(user) + if err := c.handle(sconn, chans, reqs); err != nil { + log.WithField("error", err).WithField("ip", conn.RemoteAddr().String()).Error("sftp: failed to accept inbound connection") + } + }(conn, key) + } +} + +// sessionKey returns the key used to limit the connections of a user, which is +// the user and server the Panel authenticated. +func sessionKey(sconn *ssh.ServerConn) string { + if sconn.Permissions != nil { + if user := sconn.Permissions.Extensions["user"]; user != "" { + return user + ":" + sconn.Permissions.Extensions["uuid"] + } + } + return strings.ToLower(sconn.User()) +} + +// handshake performs the SSH handshake, including authentication, on the +// connection. The connection is closed if this does not complete in time. +func handshake(conn net.Conn, config *ssh.ServerConfig, timeout time.Duration) (*ssh.ServerConn, <-chan ssh.NewChannel, <-chan *ssh.Request, error) { + if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil { + return nil, nil, nil, errors.WithStack(err) + } + sconn, chans, reqs, err := ssh.NewServerConn(conn, config) + if err != nil { + return nil, nil, nil, errors.WithStack(err) } + if err := conn.SetDeadline(time.Time{}); err != nil { + _ = sconn.Close() + return nil, nil, nil, errors.WithStack(err) + } + return sconn, chans, reqs, nil } // AcceptInbound handles an inbound connection to the instance and determines if we should // serve the request or not. func (c *SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) error { // Before beginning a handshake must be performed on the incoming net.Conn - sconn, chans, reqs, err := ssh.NewServerConn(conn, config) + sconn, chans, reqs, err := handshake(conn, config, defaultLimits.handshakeTimeout) if err != nil { - return errors.WithStack(err) + return err } + return c.handle(sconn, chans, reqs) +} + +// handle serves the channels of an authenticated connection. +func (c *SFTPServer) handle(sconn *ssh.ServerConn, chans <-chan ssh.NewChannel, reqs <-chan *ssh.Request) error { defer sconn.Close() go ssh.DiscardRequests(reqs) @@ -133,6 +348,12 @@ func (c *SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) erro continue } + srv, ok := c.manager.Get(sconn.Permissions.Extensions["uuid"]) + if !ok { + _ = ch.Reject(ssh.ConnectionFailed, "server not found") + continue + } + channel, requests, err := ch.Accept() if err != nil { continue @@ -148,10 +369,8 @@ func (c *SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) erro } }(requests) - if srv, ok := c.manager.Get(sconn.Permissions.Extensions["uuid"]); ok { - if err := c.Handle(sconn, srv, channel); err != nil { - return err - } + if err := c.Handle(sconn, srv, channel); err != nil { + return err } } @@ -169,11 +388,16 @@ func (c *SFTPServer) Handle(conn *ssh.ServerConn, srv *server.Server, channel ss ctx := srv.Sftp().Context(handler.User()) rs := sftp.NewRequestServer(channel, handler.Handlers()) + // Close the session when access is revoked, and stop waiting for that once + // the session ends on its own. + done := make(chan struct{}) + defer close(done) go func() { select { case <-ctx.Done(): srv.Log().WithField("user", conn.User()).Warn("sftp: terminating active session") _ = rs.Close() + case <-done: } }() @@ -191,7 +415,7 @@ func (c *SFTPServer) generateED25519PrivateKey() error { if err != nil { return errors.Wrap(err, "sftp: failed to generate ED25519 private key") } - if err := os.MkdirAll(path.Dir(c.PrivateKeyPath()), 0o755); err != nil { + if err := os.MkdirAll(path.Dir(c.PrivateKeyPath()), 0o700); err != nil { return errors.Wrap(err, "sftp: could not create internal sftp data directory") } o, err := os.OpenFile(c.PrivateKeyPath(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) diff --git a/sftp/server_test.go b/sftp/server_test.go new file mode 100644 index 000000000..e70c66615 --- /dev/null +++ b/sftp/server_test.go @@ -0,0 +1,201 @@ +package sftp + +import ( + "bufio" + "crypto/ed25519" + "crypto/rand" + "errors" + "net" + "os" + "strings" + "testing" + "time" + + "golang.org/x/crypto/ssh" +) + +func startTestServer(t *testing.T, limits connectionLimits) string { + t.Helper() + return startTestServerWithAuth(t, limits, func(ssh.ConnMetadata, []byte) (*ssh.Permissions, error) { + return nil, errors.New("denied") + }) +} + +func startTestServerWithAuth(t *testing.T, limits connectionLimits, auth func(ssh.ConnMetadata, []byte) (*ssh.Permissions, error)) string { + t.Helper() + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := ssh.NewSignerFromKey(key) + if err != nil { + t.Fatal(err) + } + conf := &ssh.ServerConfig{PasswordCallback: auth} + conf.AddHostKey(signer) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + srv := &SFTPServer{limits: limits} + go func() { + _ = srv.serve(listener, conf) + }() + return listener.Addr().String() +} + +// dial opens a connection from the given loopback address and reports whether +// the server accepted it, which it signals by sending its version banner. +func dial(t *testing.T, addr string, from string) (net.Conn, bool) { + t.Helper() + d := net.Dialer{LocalAddr: &net.TCPAddr{IP: net.ParseIP(from)}} + conn, err := d.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + _ = conn.SetReadDeadline(time.Now().Add(time.Second * 2)) + line, err := bufio.NewReader(conn).ReadString('\n') + _ = conn.SetReadDeadline(time.Time{}) + return conn, err == nil && strings.HasPrefix(line, "SSH-") +} + +func TestUnauthenticatedConnectionsAreClosedAfterTimeout(t *testing.T) { + addr := startTestServer(t, connectionLimits{handshakeTimeout: time.Millisecond * 250, maxPending: 128, maxPendingPerIP: 16}) + + conn, ok := dial(t, addr, "127.0.0.1") + if !ok { + t.Fatal("expected connection to be accepted") + } + + // Never send anything; the server must close the connection on its own. + start := time.Now() + _ = conn.SetReadDeadline(time.Now().Add(time.Second * 5)) + _, err := conn.Read(make([]byte, 1)) + if err == nil || errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("expected the server to close an idle unauthenticated connection, got %v after %s", err, time.Since(start)) + } +} + +func TestUnauthenticatedConnectionsAreLimited(t *testing.T) { + addr := startTestServer(t, connectionLimits{handshakeTimeout: time.Minute, maxPending: 3, maxPendingPerIP: 2}) + + // Two connections from one address are accepted, the third is not. + first, ok := dial(t, addr, "127.0.0.1") + if !ok { + t.Fatal("expected first connection to be accepted") + } + if _, ok := dial(t, addr, "127.0.0.1"); !ok { + t.Fatal("expected second connection to be accepted") + } + if _, ok := dial(t, addr, "127.0.0.1"); ok { + t.Fatal("expected third connection from the same address to be dropped") + } + + // Another address has its own limit, until the total limit is reached. + if _, ok := dial(t, addr, "127.0.0.2"); !ok { + t.Fatal("expected connection from another address to be accepted") + } + if _, ok := dial(t, addr, "127.0.0.3"); ok { + t.Fatal("expected connection to be dropped once the total limit is reached") + } + + // Closing a pending connection frees up its slot. + _ = first.Close() + deadline := time.Now().Add(time.Second * 5) + for { + if _, ok := dial(t, addr, "127.0.0.1"); ok { + break + } + if time.Now().After(deadline) { + t.Fatal("expected slot to be released when a pending connection closes") + } + time.Sleep(time.Millisecond * 50) + } +} + +// Every address in an IPv6 /64 network belongs to the same user, so they share a +// limit. +func TestConnectionKey(t *testing.T) { + tests := map[string]string{ + "127.0.0.1:2022": "127.0.0.1", + "[::ffff:10.0.0.1]:2022": "10.0.0.1", + "[2001:db8::1]:2022": "2001:db8::/64", + "[2001:db8::ffff:ffff]:2022": "2001:db8::/64", + "[2001:db8:0:1::1]:2022": "2001:db8:0:1::/64", + "[fe80::1%eth0]:2022": "fe80::/64", + "[2001:db8::abcd:1234:5678]:443": "2001:db8::/64", + } + for addr, want := range tests { + a, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + t.Fatal(err) + } + if got := connectionKey(a); got != want { + t.Errorf("connectionKey(%s) = %s, expected %s", addr, got, want) + } + } +} + +// Authenticated connections are limited per user, so that a single user cannot +// use up every connection. +func TestAuthenticatedConnectionsAreLimitedPerUser(t *testing.T) { + limits := defaultLimits + limits.maxPerUser = 2 + addr := startTestServerWithAuth(t, limits, func(ssh.ConnMetadata, []byte) (*ssh.Permissions, error) { + return &ssh.Permissions{}, nil + }) + connect := func(user string) (*ssh.Client, error) { + return ssh.Dial("tcp", addr, &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ssh.Password("password")}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: time.Second * 5, + }) + } + // closedWithin reports whether the server closes the connection in time. + closedWithin := func(c *ssh.Client, d time.Duration) bool { + done := make(chan struct{}) + go func() { + _ = c.Wait() + close(done) + }() + select { + case <-done: + return true + case <-time.After(d): + return false + } + } + + var clients []*ssh.Client + for i := 0; i < 2; i++ { + c, err := connect("user.12345678") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + clients = append(clients, c) + } + if c, err := connect("user.12345678"); err == nil { + t.Cleanup(func() { _ = c.Close() }) + if !closedWithin(c, time.Second*3) { + t.Fatal("expected a third connection for the same user to be closed") + } + } + if closedWithin(clients[0], time.Millisecond*200) { + t.Fatal("expected the connections within the limit to stay open") + } + + // Another user has their own limit. + other, err := connect("other.12345678") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = other.Close() }) + if closedWithin(other, time.Millisecond*500) { + t.Fatal("expected a connection for another user to stay open") + } +} diff --git a/sftp/session_limits_test.go b/sftp/session_limits_test.go new file mode 100644 index 000000000..25cd7a8c4 --- /dev/null +++ b/sftp/session_limits_test.go @@ -0,0 +1,266 @@ +package sftp + +import ( + "crypto/ed25519" + "crypto/rand" + "io" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/apex/log" + pkgsftp "github.com/pkg/sftp" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" + + "github.com/Rene-Roscher/wings/config" + "github.com/Rene-Roscher/wings/server" + "github.com/Rene-Roscher/wings/server/filesystem" +) + +// setSessionTestConfig installs the minimum configuration the SFTP handlers read. +func setSessionTestConfig(t *testing.T) { + t.Helper() + + config.Set(&config.Configuration{ + AuthenticationToken: "abc", + System: config.SystemConfiguration{ + RootDirectory: t.TempDir(), + DiskCheckInterval: 150, + }, + }) +} + +// startSessionHarness serves SSH connections through the SFTP server's own +// connection handler, so every channel goes through the real handle loop and +// the real Handle function. The server accepts any password. +func startSessionHarness(t *testing.T, c *SFTPServer, uuid string) string { + t.Helper() + + _, key, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + signer, err := ssh.NewSignerFromKey(key) + require.NoError(t, err) + + conf := &ssh.ServerConfig{ + PasswordCallback: func(ssh.ConnMetadata, []byte) (*ssh.Permissions, error) { + return &ssh.Permissions{Extensions: map[string]string{ + "user": "3f1c2a4e-5b6d-4e7f-8a9b-0c1d2e3f4a5b", + "uuid": uuid, + "permissions": "*", + }}, nil + }, + } + conf.AddHostKey(signer) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + sconn, chans, reqs, err := ssh.NewServerConn(conn, conf) + if err != nil { + _ = conn.Close() + return + } + _ = c.handle(sconn, chans, reqs) + }() + } + }() + + return listener.Addr().String() +} + +// dialSessionHarness opens an authenticated SSH connection to the harness. +func dialSessionHarness(t *testing.T, addr string) *ssh.Client { + t.Helper() + + client, err := ssh.Dial("tcp", addr, &ssh.ClientConfig{ + User: "user.12345678", + Auth: []ssh.AuthMethod{ssh.Password("password")}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 5 * time.Second, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + + return client +} + +// settledGoroutines waits until the goroutine count stops changing and returns it. +func settledGoroutines() int { + prev := -1 + for i := 0; i < 50; i++ { + time.Sleep(20 * time.Millisecond) + runtime.GC() + n := runtime.NumGoroutine() + if n == prev { + return n + } + prev = n + } + return prev +} + +// heapInUse returns the bytes of heap currently allocated. +func heapInUse() uint64 { + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} + +// A session that has ended releases what it held, rather than waiting for the +// user's access to be revoked. +func TestSFTPSessionsReleaseTheirWatcherWhenClientCloses(t *testing.T) { + setSessionTestConfig(t) + + srv, err := server.New(nil) + require.NoError(t, err) + t.Cleanup(func() { srv.Sftp().CancelAll() }) + + manager := server.NewEmptyManager(nil) + manager.Add(srv) + c := &SFTPServer{manager: manager} + client := dialSessionHarness(t, startSessionHarness(t, c, srv.ID())) + + openSession := func() { + t.Helper() + + sess, err := client.NewSession() + require.NoError(t, err) + require.NoError(t, sess.RequestSubsystem("sftp")) + require.NoError(t, sess.Close()) + } + + openSession() + before := settledGoroutines() + heapBefore := heapInUse() + + const sessions = 30 + for i := 0; i < sessions; i++ { + openSession() + } + after := settledGoroutines() + heapAfter := heapInUse() + + t.Logf("%d sessions ended: goroutines %d -> %d, heap in use %d -> %d bytes", sessions, before, after, heapBefore, heapAfter) + require.LessOrEqual(t, after-before, 5, "a session that has ended should not leave a goroutine running") +} + +// A channel for a server that is not registered on this node is refused. +func TestSFTPChannelsForMissingServerAreRefused(t *testing.T) { + setSessionTestConfig(t) + + c := &SFTPServer{manager: server.NewEmptyManager(nil)} + client := dialSessionHarness(t, startSessionHarness(t, c, "00000000-0000-0000-0000-000000000000")) + + before := settledGoroutines() + accepted := 0 + for i := 0; i < 30; i++ { + sess, err := client.NewSession() + if err != nil { + continue + } + accepted++ + t.Cleanup(func() { _ = sess.Close() }) + } + after := settledGoroutines() + + t.Logf("%d channels accepted for a missing server, goroutines %d -> %d", accepted, before, after) + require.Zero(t, accepted, "expected channels for a server that is not on this node to be refused") +} + +// A session can only have a limited number of files open at once, and closing a +// file frees its slot. +func TestOpenedFileHandlesAreLimitedPerSession(t *testing.T) { + setSessionTestConfig(t) + + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "file.txt"), []byte("data"), 0o644)) + fsys, err := filesystem.New(root, 0, nil) + require.NoError(t, err) + + srv, err := server.New(nil) + require.NoError(t, err) + h := &Handler{ + server: srv, + fs: fsys, + permissions: []string{"*"}, + events: &eventHandler{}, + logger: log.WithField("subsystem", "sftp"), + } + + var handles []io.Closer + t.Cleanup(func() { + for _, f := range handles { + _ = f.Close() + } + }) + + for i := 0; i < maxOpenFiles*2; i++ { + f, err := h.Fileread(&pkgsftp.Request{Filepath: "file.txt"}) + if err != nil { + break + } + handles = append(handles, f.(io.Closer)) + } + require.Len(t, handles, maxOpenFiles) + + require.NoError(t, handles[0].Close()) + f, err := h.Fileread(&pkgsftp.Request{Filepath: "file.txt"}) + require.NoError(t, err, "closing a file should free a slot") + handles[0] = f.(io.Closer) +} + +// Connections for the same account share one limit. +func TestPerUserLimitUsesAuthenticatedAccount(t *testing.T) { + limits := defaultLimits + limits.maxPerUser = 2 + addr := startTestServerWithAuth(t, limits, func(ssh.ConnMetadata, []byte) (*ssh.Permissions, error) { + return &ssh.Permissions{}, nil + }) + + connect := func(user string) (*ssh.Client, error) { + return ssh.Dial("tcp", addr, &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ssh.Password("password")}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 5 * time.Second, + }) + } + closedWithin := func(client *ssh.Client, d time.Duration) bool { + done := make(chan struct{}) + go func() { + _ = client.Wait() + close(done) + }() + select { + case <-done: + return true + case <-time.After(d): + return false + } + } + + for i := 0; i < 2; i++ { + client, err := connect("user.12345678") + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + } + + third, err := connect("USER.12345678") + if err == nil { + t.Cleanup(func() { _ = third.Close() }) + require.True(t, closedWithin(third, 3*time.Second), "a third connection for the same account should be closed") + } +} diff --git a/system/context_bag.go b/system/context_bag.go index 55016b418..6e7a7f817 100644 --- a/system/context_bag.go +++ b/system/context_bag.go @@ -29,7 +29,7 @@ func (cb *ContextBag) Context(key string) context.Context { defer cb.mu.Unlock() if _, ok := cb.items[key]; !ok { - ctx, cancel := context.WithCancel(cb.ctx) + ctx, cancel := context.WithCancel(cb.ctx) //nolint:gosec // cancel is stored and called by Cancel or CancelAll cb.items[key] = ctxHolder{ctx, cancel} } diff --git a/system/locker.go b/system/locker.go index 51b8e8efd..a9f8d3e6a 100644 --- a/system/locker.go +++ b/system/locker.go @@ -7,17 +7,23 @@ import ( "emperror.dev/errors" ) -var ErrLockerLocked = errors.Sentinel("locker: cannot acquire lock, already locked") +var ( + ErrLockerLocked = errors.Sentinel("locker: cannot acquire lock, already locked") + ErrLockerDestroyed = errors.Sentinel("locker: cannot acquire lock, locker destroyed") +) type Locker struct { mu sync.RWMutex ch chan bool + // done is closed when the locker is destroyed. + done chan struct{} } // NewLocker returns a new Locker instance. func NewLocker() *Locker { return &Locker{ - ch: make(chan bool, 1), + ch: make(chan bool, 1), + done: make(chan struct{}), } } @@ -34,6 +40,9 @@ func (l *Locker) IsLocked() bool { func (l *Locker) Acquire() error { l.mu.Lock() defer l.mu.Unlock() + if l.destroyed() { + return ErrLockerDestroyed + } select { case l.ch <- true: default: @@ -45,9 +54,26 @@ func (l *Locker) Acquire() error { // TryAcquire will attempt to acquire a power-lock until the context provided // is canceled. func (l *Locker) TryAcquire(ctx context.Context) error { + select { + case <-l.done: + return ErrLockerDestroyed + default: + } select { case l.ch <- true: + // The locker may have been destroyed while waiting. + l.mu.Lock() + defer l.mu.Unlock() + if l.destroyed() { + select { + case <-l.ch: + default: + } + return ErrLockerDestroyed + } return nil + case <-l.done: + return ErrLockerDestroyed case <-ctx.Done(): if err := ctx.Err(); err != nil { if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { @@ -70,15 +96,27 @@ func (l *Locker) Release() { l.mu.Unlock() } -// Destroy cleans up the power locker by closing the channel. +// Destroy releases the lock and stops it from being acquired again. Calling it +// more than once is a no-op. func (l *Locker) Destroy() { l.mu.Lock() - if l.ch != nil { - select { - case <-l.ch: - default: - } - close(l.ch) + defer l.mu.Unlock() + if l.destroyed() { + return + } + select { + case <-l.ch: + default: + } + close(l.done) +} + +// destroyed reports whether Destroy has been called. The caller must hold mu. +func (l *Locker) destroyed() bool { + select { + case <-l.done: + return true + default: + return false } - l.mu.Unlock() } diff --git a/system/locker_destroy_test.go b/system/locker_destroy_test.go new file mode 100644 index 000000000..b73005484 --- /dev/null +++ b/system/locker_destroy_test.go @@ -0,0 +1,57 @@ +package system + +import ( + "context" + "testing" + "time" + + "emperror.dev/errors" + "github.com/stretchr/testify/require" +) + +func TestLockerCannotBeAcquiredAfterDestroy(t *testing.T) { + l := NewLocker() + l.Destroy() + + var acquireErr, tryErr error + require.NotPanics(t, func() { + acquireErr = l.Acquire() + tryErr = l.TryAcquire(context.Background()) + }) + require.True(t, errors.Is(acquireErr, ErrLockerDestroyed)) + require.True(t, errors.Is(tryErr, ErrLockerDestroyed)) + require.False(t, l.IsLocked()) +} + +func TestLockerDestroyIsIdempotent(t *testing.T) { + l := NewLocker() + require.NoError(t, l.Acquire()) + l.Destroy() + require.NotPanics(t, l.Destroy) + + // The locker is still usable for read-only calls afterwards. + require.False(t, l.IsLocked()) + l.Release() +} + +// A caller waiting for the lock gives up when the locker is destroyed. +func TestLockerWaitersAreReleasedOnDestroy(t *testing.T) { + l := NewLocker() + require.NoError(t, l.Acquire()) + + errs := make(chan error, 1) + go func() { + errs <- l.TryAcquire(context.Background()) + }() + + time.Sleep(50 * time.Millisecond) + l.Destroy() + + select { + case err := <-errs: + require.True(t, errors.Is(err, ErrLockerDestroyed), "got %v", err) + case <-time.After(time.Second): + t.Fatal("expected the waiting caller to return once the locker was destroyed") + } + require.False(t, l.IsLocked()) +} diff --git a/system/locker_test.go b/system/locker_test.go index 0e990aa4f..2bd169a45 100644 --- a/system/locker_test.go +++ b/system/locker_test.go @@ -93,7 +93,7 @@ func TestPower(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*200) defer cancel() - l.Acquire() + g.Assert(l.Acquire()).IsNil() go func() { time.AfterFunc(time.Millisecond*50, func() { l.Release() @@ -110,7 +110,7 @@ func TestPower(t *testing.T) { g.Describe("PowerLocker#Release", func() { g.It("should release when channel is full", func() { - l.Acquire() + g.Assert(l.Acquire()).IsNil() g.Assert(l.IsLocked()).IsTrue() l.Release() g.Assert(cap(l.ch)).Equal(1) @@ -128,20 +128,18 @@ func TestPower(t *testing.T) { }) g.Describe("PowerLocker#Destroy", func() { - g.It("should unlock and close the channel", func() { - l.Acquire() + g.It("should unlock and refuse further acquires", func() { + g.Assert(l.Acquire()).IsNil() g.Assert(l.IsLocked()).IsTrue() l.Destroy() g.Assert(l.IsLocked()).IsFalse() - defer func() { - r := recover() - - g.Assert(r).IsNotNil() - g.Assert(r.(error).Error()).Equal("send on closed channel") - }() + g.Assert(errors.Is(l.Acquire(), ErrLockerDestroyed)).IsTrue() + g.Assert(errors.Is(l.TryAcquire(context.Background()), ErrLockerDestroyed)).IsTrue() + g.Assert(l.IsLocked()).IsFalse() - l.Acquire() + // Destroying it again is a no-op. + l.Destroy() }) }) }) diff --git a/system/utils.go b/system/utils.go index 1268e9217..b1e1bef01 100644 --- a/system/utils.go +++ b/system/utils.go @@ -70,7 +70,7 @@ func ScanReader(r io.Reader, callback func(line []byte)) error { // in line with that it thinks is the terminal size. Those returns break a lot of output handling, // so we'll just replace them with proper new-lines and then split it later and send each line as // its own event in the response. - line = bytes.Replace(line, cr, crr, -1) + line = bytes.ReplaceAll(line, cr, crr) ns := buf.Len() + len(line) // If the length of the line value and the current value in the buffer will diff --git a/wings-debug b/wings-debug deleted file mode 100755 index e1a0609ad..000000000 Binary files a/wings-debug and /dev/null differ diff --git a/wings-fixed b/wings-fixed deleted file mode 100755 index 35e22a943..000000000 Binary files a/wings-fixed and /dev/null differ diff --git a/wings-test b/wings-test deleted file mode 100755 index c5ab43aa7..000000000 Binary files a/wings-test and /dev/null differ diff --git a/wings.go b/wings.go index 49bf0c31a..b189c3d91 100644 --- a/wings.go +++ b/wings.go @@ -1,18 +1,10 @@ package main import ( - "math/rand" - "time" - "github.com/Rene-Roscher/wings/cmd" ) func main() { - // Since we make use of the math/rand package in the code, especially for generating - // non-cryptographically secure random strings we need to seed the RNG. Just make use - // of the current time for this. - rand.Seed(time.Now().UnixNano()) - // Execute the main binary code. cmd.Execute() }