From b2583438efefdd602cae80b21cc5aa38e7ef9b4e Mon Sep 17 00:00:00 2001 From: Teg79 Date: Wed, 18 Nov 2026 14:27:19 +0100 Subject: [PATCH 1/4] feat(komodo): prune the build cache on a schedule so Komodo's disk stays bounded A platform Stack on Komodo's LXC, urbex-build-cache, prunes every komodo.buildCache.every (24h) BuildKit's cache older than keepFor (168h) and then beyond keepStorage (3GB), and the images no container uses older than keepFor, and logs a line per run to Loki. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- internal/bootstrap/public_test.go | 14 ++++++- internal/bootstrap/stacks.go | 23 ++++++++-- internal/platform/catalog.go | 3 ++ internal/platform/config.go | 53 ++++++++++++++++++++++++ internal/platform/config_test.go | 20 +++++++++ platform/stacks/build-cache/compose.yaml | 39 +++++++++++++++++ 6 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 platform/stacks/build-cache/compose.yaml diff --git a/internal/bootstrap/public_test.go b/internal/bootstrap/public_test.go index 7861eb8..e67e39c 100644 --- a/internal/bootstrap/public_test.go +++ b/internal/bootstrap/public_test.go @@ -346,7 +346,19 @@ func TestRun_PlatformStacksFromTheGitOpsRepo(t *testing.T) { t.Errorf("stack %s tags = %v", name, fp.tags["Stack "+name]) } } - want := "urbex-keycloak,urbex-observability,urbex-tunnel," + + // The builder's cache is pruned by a Stack on Komodo's Server, with + // the defaults (urbforge/urbex-cli#92). + bc := fp.stacks["urbex-build-cache"] + if bc.ServerID != KomodoServerName || bc.RunDirectory != "platform/build-cache" { + t.Errorf("build-cache stack = %+v", bc) + } + bcEnv, _ := os.ReadFile(filepath.Join(dir, "platform/build-cache/config.env")) + for _, want := range []string{"KEEP_STORAGE=3GB", "KEEP_FOR=168h", "KEEP_EVERY=86400"} { + if !strings.Contains(string(bcEnv), want) { + t.Errorf("build-cache config.env lacks %s:\n%s", want, bcEnv) + } + } + want := "urbex-build-cache,urbex-keycloak,urbex-observability,urbex-tunnel," + "urbex-telemetry-urbex-gitea,urbex-telemetry-urbex-komodo,urbex-telemetry-urbex-technitium," + "urbex-telemetry-urbex-keycloak,urbex-telemetry-urbex-observability,urbex-telemetry-urbex-tunnel" // All at once (urbforge/urbex-cli#30): in any order. diff --git a/internal/bootstrap/stacks.go b/internal/bootstrap/stacks.go index c206f57..810f065 100644 --- a/internal/bootstrap/stacks.go +++ b/internal/bootstrap/stacks.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "sync" + "time" assets "github.com/urbforge/urbex-cli/platform" @@ -32,8 +33,11 @@ type SecretWriter interface { // secrets.sops.env; the Stack Name runs on the Server of the same name. type PlatformStack struct { Name, Dir string - Config map[string]string - Secrets map[string]string + // Server is the Komodo Server it runs on; "" is the Server of the + // same name as the Stack (its own LXC). + Server string + Config map[string]string + Secrets map[string]string } // ComposeWrapper is the command every Stack deployed from the GitOps repo @@ -96,6 +100,15 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack { stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{}, Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}}) } + // The builder's cache, pruned on Komodo's LXC (urbforge/urbex-cli#92). + bc := o.Config.Komodo.BuildCache.WithDefaults() + every, _ := time.ParseDuration(bc.Every) // validated + stacks = append(stacks, PlatformStack{Name: Hostname("build-cache"), Dir: "build-cache", Server: KomodoServerName, + Config: map[string]string{ + "KEEP_STORAGE": bc.KeepStorage, + "KEEP_FOR": bc.KeepFor, + "KEEP_EVERY": fmt.Sprint(int(every.Seconds())), + }}) return stacks } @@ -195,8 +208,12 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl } var deploys []stackDeploy for _, st := range stacks { + server := st.Server + if server == "" { + server = st.Name + } deploys = append(deploys, stackDeploy{name: st.Name, tag: PlatformTag, cfg: komodo.StackConfig{ - ServerID: st.Name, + ServerID: server, GitProvider: internal.Registry, GitAccount: GiteaAdminUser, Repo: o.Config.Git.Org + "/" + GitOpsRepoName, diff --git a/internal/platform/catalog.go b/internal/platform/catalog.go index cb382d0..6fce35e 100644 --- a/internal/platform/catalog.go +++ b/internal/platform/catalog.go @@ -127,6 +127,9 @@ var Catalog = []Service{ // cloudflared connects out to Cloudflare; its name is reserved. Endpoints: []Endpoint{{Label: "tunnel", Exposure: ExposureInternal}}, OutboundOnly: true}, + // Prunes the build cache on Komodo's LXC, the builder + // (urbforge/urbex-cli#92); it listens on nothing. + {Name: "build-cache", Kind: KindPlatform, LXC: "komodo", Deploy: DeployGitOps, Provides: []string{"build-cache-pruning"}, OutboundOnly: true}, {Name: "periphery", Kind: KindAgent, Deploy: DeployAnsible, Provides: []string{"deploy-agent"}, OutboundOnly: true}, {Name: "alloy", Kind: KindAgent, Deploy: DeployGitOps, Provides: []string{"telemetry-agent"}, OutboundOnly: true}, {Name: "quicktunnel", Kind: KindAgent, Deploy: DeployAnsible, Provides: []string{"ingress"}, OutboundOnly: true}, diff --git a/internal/platform/config.go b/internal/platform/config.go index f14f8b2..3f9c917 100644 --- a/internal/platform/config.go +++ b/internal/platform/config.go @@ -9,8 +9,10 @@ import ( "maps" "net" "os" + "regexp" "slices" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -24,8 +26,46 @@ type Config struct { DNS DNSConfig `yaml:"dns"` Keycloak KeycloakConfig `yaml:"keycloak"` Storage StorageConfig `yaml:"storage,omitempty"` + Komodo KomodoConfig `yaml:"komodo,omitempty"` } +// KomodoConfig is about Komodo, the platform's builder and deployer. +type KomodoConfig struct { + BuildCache BuildCacheConfig `yaml:"buildCache,omitempty"` +} + +// BuildCacheConfig bounds what builds leave on Komodo's disk +// (urbforge/urbex-cli#92): every Every, BuildKit's cache is pruned beyond +// KeepStorage or older than KeepFor, and so are images no container uses +// older than KeepFor. +type BuildCacheConfig struct { + KeepStorage string `yaml:"keepStorage,omitempty"` // e.g. 3GB + KeepFor string `yaml:"keepFor,omitempty"` // a duration, e.g. 168h + Every string `yaml:"every,omitempty"` // a duration, e.g. 24h +} + +// WithDefaults is b with the defaults where it sets nothing. +func (b BuildCacheConfig) WithDefaults() BuildCacheConfig { + if b.KeepStorage == "" { + b.KeepStorage = DefaultBuildCacheKeepStorage + } + if b.KeepFor == "" { + b.KeepFor = DefaultBuildCacheKeepFor + } + if b.Every == "" { + b.Every = DefaultBuildCacheEvery + } + return b +} + +// The build cache's defaults: for Komodo's 10 GiB disk, about 6 GiB of +// which the platform and a first build use. +const ( + DefaultBuildCacheKeepStorage = "3GB" + DefaultBuildCacheKeepFor = "168h" + DefaultBuildCacheEvery = "24h" +) + // StorageConfig is storage that exists outside Urbex (ADR-0028). type StorageConfig struct { // Shares are folders the operator provides - a NAS mounted on the @@ -259,6 +299,7 @@ func LoadFile(path string) (*Config, error) { } func applyDefaults(cfg *Config) { + cfg.Komodo.BuildCache = cfg.Komodo.BuildCache.WithDefaults() if cfg.Git.ForgeURL == "" { cfg.Git.ForgeURL = cfg.Git.LegacyGiteaURL } @@ -337,6 +378,18 @@ func (c *Config) Validate() []string { errs = append(errs, "storage.shares."+name+".path: "+p+" is not a folder on the Proxmox node") } } + bc := c.Komodo.BuildCache + if bc.KeepStorage != "" && !storageSize.MatchString(bc.KeepStorage) { + errs = append(errs, "komodo.buildCache.keepStorage: "+bc.KeepStorage+" is not a size such as 3GB or 500MB") + } + for field, v := range map[string]string{"keepFor": bc.KeepFor, "every": bc.Every} { + if d, err := time.ParseDuration(v); v != "" && (err != nil || d < time.Hour) { + errs = append(errs, "komodo.buildCache."+field+": "+v+" is not a duration of an hour or more, such as 24h") + } + } return errs } + +// storageSize is a size docker understands: a number and a unit. +var storageSize = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|KB|MB|GB|TB)$`) diff --git a/internal/platform/config_test.go b/internal/platform/config_test.go index ebdcdbd..3bc918b 100644 --- a/internal/platform/config_test.go +++ b/internal/platform/config_test.go @@ -269,3 +269,23 @@ func TestLoadFile_LegacyGiteaURL(t *testing.T) { } } } + +// komodo.buildCache: defaults, and what isn't a size or an hour or more +// is refused (urbforge/urbex-cli#92). +func TestBuildCacheConfig(t *testing.T) { + cfg := &Config{} + applyDefaults(cfg) + if bc := cfg.Komodo.BuildCache; bc.KeepStorage != "3GB" || bc.KeepFor != "168h" || bc.Every != "24h" { + t.Errorf("defaults = %+v", bc) + } + cfg.Komodo.BuildCache = BuildCacheConfig{KeepStorage: "lots", KeepFor: "10m", Every: "daily"} + var got []string + for _, e := range cfg.Validate() { + if strings.HasPrefix(e, "komodo.buildCache.") { + got = append(got, e) + } + } + if len(got) != 3 { + t.Errorf("errors = %v", got) + } +} diff --git a/platform/stacks/build-cache/compose.yaml b/platform/stacks/build-cache/compose.yaml new file mode 100644 index 0000000..f39fa96 --- /dev/null +++ b/platform/stacks/build-cache/compose.yaml @@ -0,0 +1,39 @@ +# Generated by urbex bootstrap from urbex-cli's platform/stacks - do not edit: +# changes are overwritten. Deployed by Komodo as the Stack urbex-build-cache, +# on Komodo's LXC, its builder (urbforge/urbex-cli#92). +# +# Every KEEP_EVERY seconds, it prunes what builds leave on the builder's +# disk: BuildKit's cache older than KEEP_FOR, then beyond KEEP_STORAGE +# (least recently used first), and images +# no container uses that are older than KEEP_FOR - the images of a +# release live in the registry, and the cache of an unchanged app stays. +# Each run's lines go to Loki, like every container's. +services: + prune: + image: docker.io/library/docker:29.9.3-cli + labels: + urbex.kind: "platform" + urbex.service: "build-cache" + restart: unless-stopped + env_file: + - config.env + volumes: + - /var/run/docker.sock:/var/run/docker.sock + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -u + sleep 600 # the first run after the builder settles + while :; do + before=$$(docker system df --format '{{.Type}} {{.Size}}' | tr '\n' ' ') + # Two passes: what is older than KEEP_FOR, then what is least + # recently used until the cache is under KEEP_STORAGE (the two + # options together would only prune the old). + old=$$(docker builder prune --force --filter "until=$$KEEP_FOR" 2>&1 | tail -n 1) + size=$$(docker builder prune --force --keep-storage "$$KEEP_STORAGE" 2>&1 | tail -n 1) + freed="older than $$KEEP_FOR: $$old, beyond $$KEEP_STORAGE: $$size" + images=$$(docker image prune --all --force --filter "until=$$KEEP_FOR" 2>&1 | tail -n 1) + after=$$(docker system df --format '{{.Type}} {{.Size}}' | tr '\n' ' ') + echo "urbex build-cache: build cache: $$freed; images: $$images; before: $$before; after: $$after" + sleep "$$KEEP_EVERY" + done From dcba2b373e3a7ad55e8326aef2b8a317bd24abf9 Mon Sep 17 00:00:00 2001 From: Teg79 Date: Wed, 18 Nov 2026 14:27:32 +0100 Subject: [PATCH 2/4] docs(changelog): the build cache's pruning Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31292a1..16d907c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,15 @@ workflow refuses a tag whose version has no entry here. ## [Unreleased] +### Added + +- **Komodo's build cache is pruned on a schedule**, so its disk stays + bounded: every day the BuildKit cache older than 7 days, then beyond + 3 GB, and the images no container uses older than 7 days. Change it + with `komodo.buildCache` in `urbex.platform.yaml`; each run logs what + it freed (Grafana, Loki). It runs as the platform Stack + `urbex-build-cache` on Komodo's LXC, created by `urbex bootstrap`. + ## [0.1.0] - 2026-11-17 The first preview release: urbex as binaries for Linux and macOS (amd64, From b98accac35f4ae408410cd83ced8b93d256e63d8 Mon Sep 17 00:00:00 2001 From: Teg79 Date: Wed, 18 Nov 2026 14:32:07 +0100 Subject: [PATCH 3/4] fix(komodo): the build cache's Stack, without secrets, runs without the compose wrapper The wrapper (sops-tools) isn't on Komodo's own LXC: a platform Stack without secrets now has no secrets file and no wrapper, as the web frontends' Stacks on the builder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- internal/bootstrap/public_test.go | 2 +- internal/bootstrap/stacks.go | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/internal/bootstrap/public_test.go b/internal/bootstrap/public_test.go index e67e39c..3c94015 100644 --- a/internal/bootstrap/public_test.go +++ b/internal/bootstrap/public_test.go @@ -349,7 +349,7 @@ func TestRun_PlatformStacksFromTheGitOpsRepo(t *testing.T) { // The builder's cache is pruned by a Stack on Komodo's Server, with // the defaults (urbforge/urbex-cli#92). bc := fp.stacks["urbex-build-cache"] - if bc.ServerID != KomodoServerName || bc.RunDirectory != "platform/build-cache" { + if bc.ServerID != KomodoServerName || bc.RunDirectory != "platform/build-cache" || bc.ComposeCmdWrapper != "" || len(bc.ConfigFiles) != 1 { t.Errorf("build-cache stack = %+v", bc) } bcEnv, _ := os.ReadFile(filepath.Join(dir, "platform/build-cache/config.env")) diff --git a/internal/bootstrap/stacks.go b/internal/bootstrap/stacks.go index 810f065..b1d7a89 100644 --- a/internal/bootstrap/stacks.go +++ b/internal/bootstrap/stacks.go @@ -168,6 +168,9 @@ func (o *Orchestrator) writePlatformStacks(ctx context.Context, gitopsRepoPath s if err := os.WriteFile(filepath.Join(dir, "config.env"), []byte(renderEnv(config)), 0o644); err != nil { return err } + if len(st.Secrets) == 0 { + continue // no secrets file, and no wrapper to decrypt it (platformStackDeploys) + } if err := store.Write(ctx, filepath.Join(dir, "secrets.sops.env"), st.Secrets); err != nil { return fmt.Errorf("encrypting the %s stack's secrets: %w", st.Dir, err) } @@ -212,7 +215,7 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl if server == "" { server = st.Name } - deploys = append(deploys, stackDeploy{name: st.Name, tag: PlatformTag, cfg: komodo.StackConfig{ + cfg := komodo.StackConfig{ ServerID: server, GitProvider: internal.Registry, GitAccount: GiteaAdminUser, @@ -222,10 +225,16 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl ProjectName: "urbex", ConfigFiles: []komodo.ConfigFile{ {Path: "config.env", Requires: "Redeploy"}, - {Path: "secrets.sops.env", Requires: "Redeploy"}, }, - ComposeCmdWrapper: ComposeWrapper, - }}) + } + // Secrets are decrypted by the compose wrapper (sops-tools); a + // Stack without them needs neither - the wrapper isn't on + // Komodo's own LXC, where the build cache's Stack runs. + if len(st.Secrets) > 0 { + cfg.ConfigFiles = append(cfg.ConfigFiles, komodo.ConfigFile{Path: "secrets.sops.env", Requires: "Redeploy"}) + cfg.ComposeCmdWrapper = ComposeWrapper + } + deploys = append(deploys, stackDeploy{name: st.Name, tag: PlatformTag, cfg: cfg}) } return deploys, nil } From 6ec301a608673737631ec6ec5614c390f0ed1bda Mon Sep 17 00:00:00 2001 From: Teg79 Date: Wed, 18 Nov 2026 14:41:52 +0100 Subject: [PATCH 4/4] test(e2e): the build cache's Stack runs on Komodo's LXC, without secrets Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0182yxsfwRowwnxCfgrQwjnA --- e2e/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e/run.sh b/e2e/run.sh index 5fd61ac..a0470bb 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -212,6 +212,11 @@ step "bootstrap: base services and platform setup" ' [ "$(komodo_stack_state urbex-telemetry-$host)" = running ] || fail "telemetry stack of $host: $(komodo_stack_state urbex-telemetry-$host)" [ "$(komodo_tags Stack urbex-telemetry-$host)" = agent ] || fail "telemetry stack of $host tags: $(komodo_tags Stack urbex-telemetry-$host)" done + # The builder'"'"'s cache is pruned by a Stack on Komodo'"'"'s LXC + # (urbforge/urbex-cli#92), with no secrets. + [ "$(komodo_stack_state urbex-build-cache)" = running ] || fail "build-cache stack: $(komodo_stack_state urbex-build-cache)" + grep -q "^KEEP_STORAGE=3GB$" /work/gitops/platform/build-cache/config.env || fail "build-cache config.env: $(cat /work/gitops/platform/build-cache/config.env)" + [ ! -e /work/gitops/platform/build-cache/secrets.sops.env ] || fail "build-cache should have no secrets file" for svc in keycloak observability; do grep -q "^sops_" /work/gitops/platform/$svc/secrets.sops.env || fail "platform/$svc/secrets.sops.env is not sops-encrypted" done @@ -407,7 +412,7 @@ step "rollback: deploying an older release" ' step "destroy staging: prod and the project Builds untouched" ' urbex destroy staging - [ "$(komodo_names Stacks)" = "hello-api-prod urbex-gitea urbex-keycloak urbex-komodo urbex-observability urbex-technitium urbex-telemetry-hello-api-prod urbex-telemetry-urbex-gitea urbex-telemetry-urbex-keycloak urbex-telemetry-urbex-komodo urbex-telemetry-urbex-observability urbex-telemetry-urbex-technitium" ] || fail "unexpected stacks left (prod and the platform'"'"'s expected): $(komodo_names Stacks)" + [ "$(komodo_names Stacks)" = "hello-api-prod urbex-build-cache urbex-gitea urbex-keycloak urbex-komodo urbex-observability urbex-technitium urbex-telemetry-hello-api-prod urbex-telemetry-urbex-gitea urbex-telemetry-urbex-keycloak urbex-telemetry-urbex-komodo urbex-telemetry-urbex-observability urbex-telemetry-urbex-technitium" ] || fail "unexpected stacks left (prod and the platform'"'"'s expected): $(komodo_names Stacks)" [ "$(komodo_names Builds)" = "hello-api" ] || fail "the project build should stay while prod exists: $(komodo_names Builds)" test ! -e /work/gitops/environments/staging/hello || fail "the staging folder is still in the gitops repo" expect_http http://$NET.211:8080/ "hello v2 from urbex (prod)"