Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ workflow refuses a tag whose version has no entry here.

## [Unreleased]

### Added

- **Komodo's build cache is pruned on a schedule**, so its disk stays
bounded: every day the BuildKit cache older than 7 days, then beyond
3 GB, and the images no container uses older than 7 days. Change it
with `komodo.buildCache` in `urbex.platform.yaml`; each run logs what
it freed (Grafana, Loki). It runs as the platform Stack
`urbex-build-cache` on Komodo's LXC, created by `urbex bootstrap`.

## [0.1.0] - 2026-11-17

The first preview release: urbex as binaries for Linux and macOS (amd64,
Expand Down
7 changes: 6 additions & 1 deletion e2e/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,11 @@ step "bootstrap: base services and platform setup" '
[ "$(komodo_stack_state urbex-telemetry-$host)" = running ] || fail "telemetry stack of $host: $(komodo_stack_state urbex-telemetry-$host)"
[ "$(komodo_tags Stack urbex-telemetry-$host)" = agent ] || fail "telemetry stack of $host tags: $(komodo_tags Stack urbex-telemetry-$host)"
done
# The builder'"'"'s cache is pruned by a Stack on Komodo'"'"'s LXC
# (urbforge/urbex-cli#92), with no secrets.
[ "$(komodo_stack_state urbex-build-cache)" = running ] || fail "build-cache stack: $(komodo_stack_state urbex-build-cache)"
grep -q "^KEEP_STORAGE=3GB$" /work/gitops/platform/build-cache/config.env || fail "build-cache config.env: $(cat /work/gitops/platform/build-cache/config.env)"
[ ! -e /work/gitops/platform/build-cache/secrets.sops.env ] || fail "build-cache should have no secrets file"
for svc in keycloak observability; do
grep -q "^sops_" /work/gitops/platform/$svc/secrets.sops.env || fail "platform/$svc/secrets.sops.env is not sops-encrypted"
done
Expand Down Expand Up @@ -407,7 +412,7 @@ step "rollback: deploying an older release" '

step "destroy staging: prod and the project Builds untouched" '
urbex destroy staging
[ "$(komodo_names Stacks)" = "hello-api-prod urbex-gitea urbex-keycloak urbex-komodo urbex-observability urbex-technitium urbex-telemetry-hello-api-prod urbex-telemetry-urbex-gitea urbex-telemetry-urbex-keycloak urbex-telemetry-urbex-komodo urbex-telemetry-urbex-observability urbex-telemetry-urbex-technitium" ] || fail "unexpected stacks left (prod and the platform'"'"'s expected): $(komodo_names Stacks)"
[ "$(komodo_names Stacks)" = "hello-api-prod urbex-build-cache urbex-gitea urbex-keycloak urbex-komodo urbex-observability urbex-technitium urbex-telemetry-hello-api-prod urbex-telemetry-urbex-gitea urbex-telemetry-urbex-keycloak urbex-telemetry-urbex-komodo urbex-telemetry-urbex-observability urbex-telemetry-urbex-technitium" ] || fail "unexpected stacks left (prod and the platform'"'"'s expected): $(komodo_names Stacks)"
[ "$(komodo_names Builds)" = "hello-api" ] || fail "the project build should stay while prod exists: $(komodo_names Builds)"
test ! -e /work/gitops/environments/staging/hello || fail "the staging folder is still in the gitops repo"
expect_http http://$NET.211:8080/ "hello v2 from urbex (prod)"
Expand Down
14 changes: 13 additions & 1 deletion internal/bootstrap/public_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,19 @@ func TestRun_PlatformStacksFromTheGitOpsRepo(t *testing.T) {
t.Errorf("stack %s tags = %v", name, fp.tags["Stack "+name])
}
}
want := "urbex-keycloak,urbex-observability,urbex-tunnel," +
// The builder's cache is pruned by a Stack on Komodo's Server, with
// the defaults (urbforge/urbex-cli#92).
bc := fp.stacks["urbex-build-cache"]
if bc.ServerID != KomodoServerName || bc.RunDirectory != "platform/build-cache" || bc.ComposeCmdWrapper != "" || len(bc.ConfigFiles) != 1 {
t.Errorf("build-cache stack = %+v", bc)
}
bcEnv, _ := os.ReadFile(filepath.Join(dir, "platform/build-cache/config.env"))
for _, want := range []string{"KEEP_STORAGE=3GB", "KEEP_FOR=168h", "KEEP_EVERY=86400"} {
if !strings.Contains(string(bcEnv), want) {
t.Errorf("build-cache config.env lacks %s:\n%s", want, bcEnv)
}
}
want := "urbex-build-cache,urbex-keycloak,urbex-observability,urbex-tunnel," +
"urbex-telemetry-urbex-gitea,urbex-telemetry-urbex-komodo,urbex-telemetry-urbex-technitium," +
"urbex-telemetry-urbex-keycloak,urbex-telemetry-urbex-observability,urbex-telemetry-urbex-tunnel"
// All at once (urbforge/urbex-cli#30): in any order.
Expand Down
40 changes: 33 additions & 7 deletions internal/bootstrap/stacks.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"sort"
"strings"
"sync"
"time"

assets "github.com/urbforge/urbex-cli/platform"

Expand All @@ -32,8 +33,11 @@ type SecretWriter interface {
// secrets.sops.env; the Stack Name runs on the Server of the same name.
type PlatformStack struct {
Name, Dir string
Config map[string]string
Secrets map[string]string
// Server is the Komodo Server it runs on; "" is the Server of the
// same name as the Stack (its own LXC).
Server string
Config map[string]string
Secrets map[string]string
}

// ComposeWrapper is the command every Stack deployed from the GitOps repo
Expand Down Expand Up @@ -96,6 +100,15 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack {
stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{},
Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}})
}
// The builder's cache, pruned on Komodo's LXC (urbforge/urbex-cli#92).
bc := o.Config.Komodo.BuildCache.WithDefaults()
every, _ := time.ParseDuration(bc.Every) // validated
stacks = append(stacks, PlatformStack{Name: Hostname("build-cache"), Dir: "build-cache", Server: KomodoServerName,
Config: map[string]string{
"KEEP_STORAGE": bc.KeepStorage,
"KEEP_FOR": bc.KeepFor,
"KEEP_EVERY": fmt.Sprint(int(every.Seconds())),
}})
return stacks
}

Expand Down Expand Up @@ -155,6 +168,9 @@ func (o *Orchestrator) writePlatformStacks(ctx context.Context, gitopsRepoPath s
if err := os.WriteFile(filepath.Join(dir, "config.env"), []byte(renderEnv(config)), 0o644); err != nil {
return err
}
if len(st.Secrets) == 0 {
continue // no secrets file, and no wrapper to decrypt it (platformStackDeploys)
}
if err := store.Write(ctx, filepath.Join(dir, "secrets.sops.env"), st.Secrets); err != nil {
return fmt.Errorf("encrypting the %s stack's secrets: %w", st.Dir, err)
}
Expand Down Expand Up @@ -195,8 +211,12 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl
}
var deploys []stackDeploy
for _, st := range stacks {
deploys = append(deploys, stackDeploy{name: st.Name, tag: PlatformTag, cfg: komodo.StackConfig{
ServerID: st.Name,
server := st.Server
if server == "" {
server = st.Name
}
cfg := komodo.StackConfig{
ServerID: server,
GitProvider: internal.Registry,
GitAccount: GiteaAdminUser,
Repo: o.Config.Git.Org + "/" + GitOpsRepoName,
Expand All @@ -205,10 +225,16 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl
ProjectName: "urbex",
ConfigFiles: []komodo.ConfigFile{
{Path: "config.env", Requires: "Redeploy"},
{Path: "secrets.sops.env", Requires: "Redeploy"},
},
ComposeCmdWrapper: ComposeWrapper,
}})
}
// Secrets are decrypted by the compose wrapper (sops-tools); a
// Stack without them needs neither - the wrapper isn't on
// Komodo's own LXC, where the build cache's Stack runs.
if len(st.Secrets) > 0 {
cfg.ConfigFiles = append(cfg.ConfigFiles, komodo.ConfigFile{Path: "secrets.sops.env", Requires: "Redeploy"})
cfg.ComposeCmdWrapper = ComposeWrapper
}
deploys = append(deploys, stackDeploy{name: st.Name, tag: PlatformTag, cfg: cfg})
}
return deploys, nil
}
Expand Down
3 changes: 3 additions & 0 deletions internal/platform/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@ var Catalog = []Service{
// cloudflared connects out to Cloudflare; its name is reserved.
Endpoints: []Endpoint{{Label: "tunnel", Exposure: ExposureInternal}},
OutboundOnly: true},
// Prunes the build cache on Komodo's LXC, the builder
// (urbforge/urbex-cli#92); it listens on nothing.
{Name: "build-cache", Kind: KindPlatform, LXC: "komodo", Deploy: DeployGitOps, Provides: []string{"build-cache-pruning"}, OutboundOnly: true},
{Name: "periphery", Kind: KindAgent, Deploy: DeployAnsible, Provides: []string{"deploy-agent"}, OutboundOnly: true},
{Name: "alloy", Kind: KindAgent, Deploy: DeployGitOps, Provides: []string{"telemetry-agent"}, OutboundOnly: true},
{Name: "quicktunnel", Kind: KindAgent, Deploy: DeployAnsible, Provides: []string{"ingress"}, OutboundOnly: true},
Expand Down
53 changes: 53 additions & 0 deletions internal/platform/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import (
"maps"
"net"
"os"
"regexp"
"slices"
"strings"
"time"

"gopkg.in/yaml.v3"
)
Expand All @@ -24,8 +26,46 @@ type Config struct {
DNS DNSConfig `yaml:"dns"`
Keycloak KeycloakConfig `yaml:"keycloak"`
Storage StorageConfig `yaml:"storage,omitempty"`
Komodo KomodoConfig `yaml:"komodo,omitempty"`
}

// KomodoConfig is about Komodo, the platform's builder and deployer.
type KomodoConfig struct {
BuildCache BuildCacheConfig `yaml:"buildCache,omitempty"`
}

// BuildCacheConfig bounds what builds leave on Komodo's disk
// (urbforge/urbex-cli#92): every Every, BuildKit's cache is pruned beyond
// KeepStorage or older than KeepFor, and so are images no container uses
// older than KeepFor.
type BuildCacheConfig struct {
KeepStorage string `yaml:"keepStorage,omitempty"` // e.g. 3GB
KeepFor string `yaml:"keepFor,omitempty"` // a duration, e.g. 168h
Every string `yaml:"every,omitempty"` // a duration, e.g. 24h
}

// WithDefaults is b with the defaults where it sets nothing.
func (b BuildCacheConfig) WithDefaults() BuildCacheConfig {
if b.KeepStorage == "" {
b.KeepStorage = DefaultBuildCacheKeepStorage
}
if b.KeepFor == "" {
b.KeepFor = DefaultBuildCacheKeepFor
}
if b.Every == "" {
b.Every = DefaultBuildCacheEvery
}
return b
}

// The build cache's defaults: for Komodo's 10 GiB disk, about 6 GiB of
// which the platform and a first build use.
const (
DefaultBuildCacheKeepStorage = "3GB"
DefaultBuildCacheKeepFor = "168h"
DefaultBuildCacheEvery = "24h"
)

// StorageConfig is storage that exists outside Urbex (ADR-0028).
type StorageConfig struct {
// Shares are folders the operator provides - a NAS mounted on the
Expand Down Expand Up @@ -259,6 +299,7 @@ func LoadFile(path string) (*Config, error) {
}

func applyDefaults(cfg *Config) {
cfg.Komodo.BuildCache = cfg.Komodo.BuildCache.WithDefaults()
if cfg.Git.ForgeURL == "" {
cfg.Git.ForgeURL = cfg.Git.LegacyGiteaURL
}
Expand Down Expand Up @@ -337,6 +378,18 @@ func (c *Config) Validate() []string {
errs = append(errs, "storage.shares."+name+".path: "+p+" is not a folder on the Proxmox node")
}
}
bc := c.Komodo.BuildCache
if bc.KeepStorage != "" && !storageSize.MatchString(bc.KeepStorage) {
errs = append(errs, "komodo.buildCache.keepStorage: "+bc.KeepStorage+" is not a size such as 3GB or 500MB")
}
for field, v := range map[string]string{"keepFor": bc.KeepFor, "every": bc.Every} {
if d, err := time.ParseDuration(v); v != "" && (err != nil || d < time.Hour) {
errs = append(errs, "komodo.buildCache."+field+": "+v+" is not a duration of an hour or more, such as 24h")
}
}

return errs
}

// storageSize is a size docker understands: a number and a unit.
var storageSize = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(B|KB|MB|GB|TB)$`)
20 changes: 20 additions & 0 deletions internal/platform/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,3 +269,23 @@ func TestLoadFile_LegacyGiteaURL(t *testing.T) {
}
}
}

// komodo.buildCache: defaults, and what isn't a size or an hour or more
// is refused (urbforge/urbex-cli#92).
func TestBuildCacheConfig(t *testing.T) {
cfg := &Config{}
applyDefaults(cfg)
if bc := cfg.Komodo.BuildCache; bc.KeepStorage != "3GB" || bc.KeepFor != "168h" || bc.Every != "24h" {
t.Errorf("defaults = %+v", bc)
}
cfg.Komodo.BuildCache = BuildCacheConfig{KeepStorage: "lots", KeepFor: "10m", Every: "daily"}
var got []string
for _, e := range cfg.Validate() {
if strings.HasPrefix(e, "komodo.buildCache.") {
got = append(got, e)
}
}
if len(got) != 3 {
t.Errorf("errors = %v", got)
}
}
39 changes: 39 additions & 0 deletions platform/stacks/build-cache/compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Generated by urbex bootstrap from urbex-cli's platform/stacks - do not edit:
# changes are overwritten. Deployed by Komodo as the Stack urbex-build-cache,
# on Komodo's LXC, its builder (urbforge/urbex-cli#92).
#
# Every KEEP_EVERY seconds, it prunes what builds leave on the builder's
# disk: BuildKit's cache older than KEEP_FOR, then beyond KEEP_STORAGE
# (least recently used first), and images
# no container uses that are older than KEEP_FOR - the images of a
# release live in the registry, and the cache of an unchanged app stays.
# Each run's lines go to Loki, like every container's.
services:
prune:
image: docker.io/library/docker:29.9.3-cli
labels:
urbex.kind: "platform"
urbex.service: "build-cache"
restart: unless-stopped
env_file:
- config.env
volumes:
- /var/run/docker.sock:/var/run/docker.sock
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -u
sleep 600 # the first run after the builder settles
while :; do
before=$$(docker system df --format '{{.Type}} {{.Size}}' | tr '\n' ' ')
# Two passes: what is older than KEEP_FOR, then what is least
# recently used until the cache is under KEEP_STORAGE (the two
# options together would only prune the old).
old=$$(docker builder prune --force --filter "until=$$KEEP_FOR" 2>&1 | tail -n 1)
size=$$(docker builder prune --force --keep-storage "$$KEEP_STORAGE" 2>&1 | tail -n 1)
freed="older than $$KEEP_FOR: $$old, beyond $$KEEP_STORAGE: $$size"
images=$$(docker image prune --all --force --filter "until=$$KEEP_FOR" 2>&1 | tail -n 1)
after=$$(docker system df --format '{{.Type}} {{.Size}}' | tr '\n' ' ')
echo "urbex build-cache: build cache: $$freed; images: $$images; before: $$before; after: $$after"
sleep "$$KEEP_EVERY"
done