From 581aedf5c2751a3316f087c81e75e3b040c0ab0f Mon Sep 17 00:00:00 2001 From: l-m Date: Wed, 11 Nov 2026 20:28:06 +1100 Subject: [PATCH 1/3] store: make the SQLite journal mode configurable --- README.md | 1 + cmd/opence-fleet/main.go | 10 +++++----- internal/config/config.go | 34 +++++++++++++++++++++------------- internal/config/config_test.go | 27 +++++++++++++++++++++++++++ internal/store/store.go | 23 ++++++++++++++++++++--- internal/store/store_test.go | 20 ++++++++++++++++++++ 6 files changed, 94 insertions(+), 21 deletions(-) create mode 100644 internal/config/config_test.go diff --git a/README.md b/README.md index ccdbe10..5a5bc5b 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ overrides it. | `FLEET_PUBLIC_URL` | `--public-url` | — | the fleet's public HTTPS origin (https://fleet.example.org), for the agents' one-liner and links | | `FLEET_LISTEN` | `--listen` | `127.0.0.1:8070` | the HTTP listener's address | | `FLEET_DB` | `--db` | — | the SQLite database file (default: fleet.db in FLEET_DATA) | +| `FLEET_DB_JOURNAL` | `--db-journal` | `wal` | the SQLite journal mode: `wal`, `delete`, `truncate` or `persist`; use `delete` or `truncate` on network filesystems where WAL locking is unsafe | | `FLEET_DATA` | `--data` | `fleet-data` | the folder of the fleet's files: the library's blobs, the releases' zips, fleet.key, bin/ | | `FLEET_SECRET_KEY` | `--secret-key` | — | the key that encrypts secrets at rest (made if missing; default: fleet.key in FLEET_DATA) | | `FLEET_TRUSTED_PROXIES` | `--trusted-proxies` | — | reverse proxies (addresses or CIDRs, commas) whose X-Forwarded-* are believed | diff --git a/cmd/opence-fleet/main.go b/cmd/opence-fleet/main.go index 7e353bf..a6bed6d 100644 --- a/cmd/opence-fleet/main.go +++ b/cmd/opence-fleet/main.go @@ -159,7 +159,7 @@ func openFleet(args []string, name string) (*config.ServeConfig, *store.Store, * if err := os.MkdirAll(c.Data, 0o700); err != nil { return nil, nil, nil, err } - s, err := store.Open(c.DB) + s, err := store.OpenWithJournal(c.DB, c.DBJournal) if err != nil { return nil, nil, nil, err } @@ -210,7 +210,7 @@ func serve(args []string) error { if err := update.BackupDatabase(c.DB, version, log); err != nil { return err } - s, err := store.Open(c.DB) + s, err := store.OpenWithJournal(c.DB, c.DBJournal) if err != nil { return fmt.Errorf("the database %s: %w", c.DB, err) } @@ -471,7 +471,7 @@ func joinToken(args []string) error { if err != nil { return err } - s, err := store.Open(c.DB) + s, err := store.OpenWithJournal(c.DB, c.DBJournal) if err != nil { return err } @@ -522,7 +522,7 @@ func releaseZip(args []string) error { if err != nil { return err } - s, err := store.Open(c.DB) + s, err := store.OpenWithJournal(c.DB, c.DBJournal) if err != nil { return err } @@ -583,7 +583,7 @@ func resetPassword(args []string) error { if err != nil { return err } - s, err := store.Open(c.DB) + s, err := store.OpenWithJournal(c.DB, c.DBJournal) if err != nil { return err } diff --git a/internal/config/config.go b/internal/config/config.go index c52cfa6..9db2668 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ var Serve = []Option{ {Env: "FLEET_PUBLIC_URL", Flag: "public-url", Help: "the fleet's public HTTPS origin (https://fleet.example.org), for the agents' one-liner and links"}, {Env: "FLEET_LISTEN", Flag: "listen", Help: "the HTTP listener's address", Default: "127.0.0.1:8070"}, {Env: "FLEET_DB", Flag: "db", Help: "the SQLite database file (default: fleet.db in FLEET_DATA)"}, + {Env: "FLEET_DB_JOURNAL", Flag: "db-journal", Help: "the SQLite journal mode: wal, delete, truncate or persist (use delete or truncate on network filesystems where WAL locking is unsafe)", Default: "wal"}, {Env: "FLEET_DATA", Flag: "data", Help: "the folder of the fleet's files: the library's blobs, the releases' zips, fleet.key, bin/", Default: "fleet-data"}, {Env: "FLEET_SECRET_KEY", Flag: "secret-key", Help: "the key that encrypts secrets at rest (made if missing; default: fleet.key in FLEET_DATA)"}, {Env: "FLEET_TRUSTED_PROXIES", Flag: "trusted-proxies", Help: "reverse proxies (addresses or CIDRs, commas) whose X-Forwarded-* are believed"}, @@ -139,25 +140,25 @@ func Bool(text string) (bool, error) { // ServeConfig is the control plane's options, read. type ServeConfig struct { - Name, PublicURL, Listen, DB, Data, SecretKey string - TrustedProxies []netip.Prefix - TLSCert, TLSKey, MastersFile string - Image, ReleaseRepository string - ReleaseCheckInterval time.Duration - EventsDays, SamplesDays int - LogAddresses, PublicStatus, Metrics bool - InsecureHTTP bool - LogLevel, Updates, UpdateURL, UpdateKey string - Timezone *time.Location - CAFile string - SessionHours, SessionIdleMinutes int + Name, PublicURL, Listen, DB, DBJournal, Data, SecretKey string + TrustedProxies []netip.Prefix + TLSCert, TLSKey, MastersFile string + Image, ReleaseRepository string + ReleaseCheckInterval time.Duration + EventsDays, SamplesDays int + LogAddresses, PublicStatus, Metrics bool + InsecureHTTP bool + LogLevel, Updates, UpdateURL, UpdateKey string + Timezone *time.Location + CAFile string + SessionHours, SessionIdleMinutes int } // LoadServe reads the control plane's options. func LoadServe(v Values) (*ServeConfig, error) { c := &ServeConfig{ Name: v.Get("FLEET_NAME"), PublicURL: strings.TrimRight(v.Get("FLEET_PUBLIC_URL"), "/"), - Listen: v.Get("FLEET_LISTEN"), DB: v.Get("FLEET_DB"), Data: v.Get("FLEET_DATA"), + Listen: v.Get("FLEET_LISTEN"), DB: v.Get("FLEET_DB"), DBJournal: strings.ToLower(v.Get("FLEET_DB_JOURNAL")), Data: v.Get("FLEET_DATA"), SecretKey: v.Get("FLEET_SECRET_KEY"), TLSCert: v.Get("FLEET_TLS_CERT"), TLSKey: v.Get("FLEET_TLS_KEY"), MastersFile: v.Get("FLEET_MASTERS_FILE"), Image: strings.TrimRight(v.Get("FLEET_IMAGE"), "/"), ReleaseRepository: v.Get("FLEET_RELEASE_REPOSITORY"), LogLevel: strings.ToLower(v.Get("FLEET_LOG_LEVEL")), @@ -177,6 +178,13 @@ func LoadServe(v Values) (*ServeConfig, error) { if c.DB == "" { c.DB = filepath.Join(c.Data, "fleet.db") } + switch c.DBJournal { + case "": + c.DBJournal = "wal" + case "wal", "delete", "truncate", "persist": + default: + errs = append(errs, errors.New("FLEET_DB_JOURNAL: wal, delete, truncate or persist")) + } if c.SecretKey == "" { c.SecretKey = filepath.Join(c.Data, "fleet.key") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..b3f38d7 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "flag" + "testing" +) + +func TestLoadServeJournal(t *testing.T) { + load := func(value string) (*ServeConfig, error) { + set := flag.NewFlagSet("serve", flag.ContinueOnError) + values := Flags(set, Serve) + if err := set.Parse([]string{"--db-journal", value}); err != nil { + t.Fatal(err) + } + return LoadServe(values) + } + c, err := load("delete") + if err != nil { + t.Fatal(err) + } + if c.DBJournal != "delete" { + t.Fatalf("journal = %q", c.DBJournal) + } + if _, err := load("not-a-journal"); err == nil { + t.Fatal("invalid journal mode accepted") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 8747ad5..fbda023 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -35,9 +35,26 @@ type Store struct { Path string } -// Open opens (making it, mode 0600) the database and brings its schema up -// to date. +// Open opens (making it, mode 0600) the database with the default WAL +// journal and brings its schema up to date. func Open(path string) (*Store, error) { + return OpenWithJournal(path, "wal") +} + +// OpenWithJournal opens (making it, mode 0600) the database with a journal +// mode of wal, delete, truncate or persist and brings its schema up to date. +// delete and truncate are the safe choices on network filesystems, where WAL +// locking is not reliable. +func OpenWithJournal(path, journal string) (*Store, error) { + journal = strings.ToLower(strings.TrimSpace(journal)) + if journal == "" { + journal = "wal" + } + switch journal { + case "wal", "delete", "truncate", "persist": + default: + return nil, fmt.Errorf("journal mode %q: wal, delete, truncate or persist", journal) + } memory := path == ":memory:" || strings.HasPrefix(path, "file::memory:") if !memory { if dir := filepath.Dir(path); dir != "" { @@ -55,7 +72,7 @@ func Open(path string) (*Store, error) { // (one connection: each would be its own empty database) dsn = "file::memory:?_pragma=foreign_keys(1)&_txlock=immediate" } else { - dsn += "&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + dsn += "&_pragma=journal_mode(" + strings.ToUpper(journal) + ")&_pragma=synchronous(NORMAL)" } conn, err := sql.Open("sqlite", dsn) if err != nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index e535770..3081200 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "os" "path/filepath" + "strings" "testing" "time" @@ -146,3 +147,22 @@ func TestUpdatePolicyNewFleet(t *testing.T) { } } } + +func TestOpenWithJournal(t *testing.T) { + path := filepath.Join(t.TempDir(), "fleet.db") + s, err := OpenWithJournal(path, "delete") + if err != nil { + t.Fatal(err) + } + var mode string + if err := s.DB.QueryRow(`PRAGMA journal_mode`).Scan(&mode); err != nil { + t.Fatal(err) + } + if strings.ToLower(mode) != "delete" { + t.Fatalf("journal mode = %q", mode) + } + s.Close() + if _, err := OpenWithJournal(path, "not-a-journal"); err == nil { + t.Fatal("invalid journal mode accepted") + } +} From e035b06e80d290c430d39e55d8be995f851d00ed Mon Sep 17 00:00:00 2001 From: l-m Date: Wed, 11 Nov 2026 20:32:24 +1100 Subject: [PATCH 2/3] store: use FULL synchronous with rollback journals --- README.md | 2 +- internal/store/store.go | 6 +++++- internal/store/store_test.go | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5a5bc5b..70addcf 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ overrides it. | `FLEET_PUBLIC_URL` | `--public-url` | — | the fleet's public HTTPS origin (https://fleet.example.org), for the agents' one-liner and links | | `FLEET_LISTEN` | `--listen` | `127.0.0.1:8070` | the HTTP listener's address | | `FLEET_DB` | `--db` | — | the SQLite database file (default: fleet.db in FLEET_DATA) | -| `FLEET_DB_JOURNAL` | `--db-journal` | `wal` | the SQLite journal mode: `wal`, `delete`, `truncate` or `persist`; use `delete` or `truncate` on network filesystems where WAL locking is unsafe | +| `FLEET_DB_JOURNAL` | `--db-journal` | `wal` | the SQLite journal mode: `wal`, `delete`, `truncate` or `persist`; use `delete` or `truncate` on network filesystems where WAL locking is unsafe; those modes use `synchronous=FULL` | | `FLEET_DATA` | `--data` | `fleet-data` | the folder of the fleet's files: the library's blobs, the releases' zips, fleet.key, bin/ | | `FLEET_SECRET_KEY` | `--secret-key` | — | the key that encrypts secrets at rest (made if missing; default: fleet.key in FLEET_DATA) | | `FLEET_TRUSTED_PROXIES` | `--trusted-proxies` | — | reverse proxies (addresses or CIDRs, commas) whose X-Forwarded-* are believed | diff --git a/internal/store/store.go b/internal/store/store.go index fbda023..582b3b2 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -72,7 +72,11 @@ func OpenWithJournal(path, journal string) (*Store, error) { // (one connection: each would be its own empty database) dsn = "file::memory:?_pragma=foreign_keys(1)&_txlock=immediate" } else { - dsn += "&_pragma=journal_mode(" + strings.ToUpper(journal) + ")&_pragma=synchronous(NORMAL)" + synchronous := "NORMAL" + if journal != "wal" { + synchronous = "FULL" + } + dsn += "&_pragma=journal_mode(" + strings.ToUpper(journal) + ")&_pragma=synchronous(" + synchronous + ")" } conn, err := sql.Open("sqlite", dsn) if err != nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3081200..a256857 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -161,6 +161,13 @@ func TestOpenWithJournal(t *testing.T) { if strings.ToLower(mode) != "delete" { t.Fatalf("journal mode = %q", mode) } + var synchronous int + if err := s.DB.QueryRow(`PRAGMA synchronous`).Scan(&synchronous); err != nil { + t.Fatal(err) + } + if synchronous != 2 { + t.Fatalf("synchronous = %d, want FULL (2)", synchronous) + } s.Close() if _, err := OpenWithJournal(path, "not-a-journal"); err == nil { t.Fatal("invalid journal mode accepted") From 334bc9f9d1fb97d339d411c57c00ab71995ede5f Mon Sep 17 00:00:00 2001 From: l-m Date: Wed, 11 Nov 2026 20:35:07 +1100 Subject: [PATCH 3/3] store: assert rollback journals leave no WAL sidecars --- internal/store/store_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index a256857..e9e59cb 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -169,6 +169,11 @@ func TestOpenWithJournal(t *testing.T) { t.Fatalf("synchronous = %d, want FULL (2)", synchronous) } s.Close() + for _, suffix := range []string{"-wal", "-shm"} { + if _, err := os.Stat(path + suffix); !os.IsNotExist(err) { + t.Fatalf("rollback journal left %s: %v", suffix, err) + } + } if _, err := OpenWithJournal(path, "not-a-journal"); err == nil { t.Fatal("invalid journal mode accepted") }