Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@ workflow refuses a tag whose version has no entry here.

## [Unreleased]

### Added

- **The Proxmox node in Grafana** (#94): with an optional read-only token
(`proxmoxMonitorToken`, role `PVEAuditor`), `urbex bootstrap` runs
`pve-exporter` in the observability Stack. The *Urbex resources*
dashboard then shows the node's CPU, memory (guests outside urbex and
ZFS's ARC included, as Proxmox counts it) and each storage's use, and
Grafana alerts when the node's memory stays above 90% or a storage
passes 85% / 95%. Only the node and storage series are kept, never other
guests'. `urbex doctor` checks the token, or says the node metrics are
off without it.

## [0.1.0] - 2026-11-17

The first preview release: urbex as binaries for Linux and macOS (amd64,
Expand Down
5 changes: 4 additions & 1 deletion internal/bootstrap/orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,10 @@ func (o *Orchestrator) Run(ctx context.Context, opts Options) error {
// The platform services Komodo deploys from the GitOps repo: their
// folders go in before setupPlatform pushes it.
o.Progress.Begin(PhaseGitOps)
stacks := o.platformStacks(tunnelToken)
stacks, err := o.platformStacks(tunnelToken)
if err != nil {
return err
}
if err := o.writePlatformStacks(ctx, opts.GitOpsRepoPath, stacks); err != nil {
return err
}
Expand Down
53 changes: 53 additions & 0 deletions internal/bootstrap/public_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,3 +463,56 @@ func TestBaseServicesAreInTheCatalog(t *testing.T) {
}
}
}

// The node's metrics (urbforge/urbex-cli#94): with the read-only token,
// pve-exporter's profile is on, reaching the node by its address, the
// token's parts in the Stack's secrets; without it, the profile is
// explicitly off.
func TestRun_NodeMetricsWithTheMonitoringToken(t *testing.T) {
dir := t.TempDir()
o, _, fp := newRunOrchestrator(t, &fakeProxmox{})
o.Config.Proxmox.APIURL = "https://192.0.2.10:8006/api2/json"
o.Credentials.ProxmoxMonitorToken = "urbex-monitor@pve!metrics=00000000-1111-2222-3333-444444444444"
if err := o.Run(context.Background(), Options{GitOpsRepoPath: dir}); err != nil {
t.Fatalf("Run: %v", err)
}
if env := fp.stacks["urbex-observability"].Environment; env != "COMPOSE_PROFILES=pve\nURBEX_PVE_NODE_IP=192.0.2.10\n" {
t.Errorf("observability environment = %q", env)
}
secrets, _ := os.ReadFile(filepath.Join(dir, "platform/observability/secrets.sops.env"))
for _, want := range []string{"PVE_USER=urbex-monitor@pve", "PVE_TOKEN_NAME=metrics", "PVE_TOKEN_VALUE=00000000-1111-2222-3333-444444444444"} {
if !strings.Contains(string(secrets), want) {
t.Errorf("secrets lack %s:\n%s", want, secrets)
}
}
config, _ := os.ReadFile(filepath.Join(dir, "platform/observability/config.env"))
if strings.Contains(string(config), "00000000-1111") || !strings.Contains(string(config), "PVE_VERIFY_SSL=false") {
t.Errorf("config.env:\n%s", config)
}

o2, _, fp2 := newRunOrchestrator(t, &fakeProxmox{})
if err := o2.Run(context.Background(), Options{GitOpsRepoPath: t.TempDir()}); err != nil {
t.Fatalf("Run without the token: %v", err)
}
if env := fp2.stacks["urbex-observability"].Environment; env != "COMPOSE_PROFILES=\nURBEX_PVE_NODE_IP=\n" {
t.Errorf("without the token, environment = %q", env)
}

o3, _, _ := newRunOrchestrator(t, &fakeProxmox{})
o3.Credentials.ProxmoxMonitorToken = "not-a-token"
if err := o3.Run(context.Background(), Options{GitOpsRepoPath: t.TempDir()}); err == nil || !strings.Contains(err.Error(), "proxmoxMonitorToken") {
t.Errorf("a malformed token: %v", err)
}
}

func TestSplitProxmoxToken(t *testing.T) {
u, n, v, err := SplitProxmoxToken("urbex@pve!urbex=a-b=c")
if err != nil || u != "urbex@pve" || n != "urbex" || v != "a-b=c" {
t.Errorf("got %q %q %q %v", u, n, v, err)
}
for _, bad := range []string{"", "urbex@pve!urbex", "urbex!x=y", "@pve!x=y", "u@pve!=y", "u@pve!x="} {
if _, _, _, err := SplitProxmoxToken(bad); err == nil {
t.Errorf("%q accepted", bad)
}
}
}
74 changes: 70 additions & 4 deletions internal/bootstrap/stacks.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"errors"
"fmt"
"io/fs"
"net"
"net/url"
"os"
"path"
"path/filepath"
Expand Down Expand Up @@ -34,6 +36,9 @@ type PlatformStack struct {
Name, Dir string
Config map[string]string
Secrets map[string]string
// Environment is the Stack's .env, for compose itself (profiles,
// interpolation), set in Komodo; "" leaves it empty.
Environment string
}

// ComposeWrapper is the command every Stack deployed from the GitOps repo
Expand All @@ -52,7 +57,7 @@ const PlatformStacksDir = "platform"
// platformStacks are the platform services deployed from the GitOps
// repo: Keycloak, the observability stack and, with Cloudflare, the
// tunnel, whose connector token is tunnelToken.
func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack {
func (o *Orchestrator) platformStacks(tunnelToken string) ([]PlatformStack, error) {
// Keycloak runs in production mode on its own PostgreSQL, in the same
// Stack (urbex-cli#36). Both containers read config.env and the
// secrets: KC_* configure Keycloak, POSTGRES_* the database.
Expand Down Expand Up @@ -89,14 +94,74 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack {
"KC_DB_PASSWORD": o.Credentials.KeycloakDBPassword,
"POSTGRES_PASSWORD": o.Credentials.KeycloakDBPassword,
}},
{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{},
Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}},
}
observability, err := o.observabilityStack()
if err != nil {
return nil, err
}
stacks = append(stacks, observability)
if o.Config.Cloudflare.Enabled() {
stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{},
Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}})
}
return stacks
return stacks, nil
}

// observabilityStack is Prometheus, Loki and Grafana; with the node's
// read-only token, also pve-exporter for the node's metrics
// (urbforge/urbex-cli#94).
func (o *Orchestrator) observabilityStack() (PlatformStack, error) {
st := PlatformStack{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{},
Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword},
// Written even without the token, so that removing it turns the
// exporter off.
Environment: "COMPOSE_PROFILES=\nURBEX_PVE_NODE_IP=\n"}
if o.Credentials.ProxmoxMonitorToken == "" {
return st, nil
}
user, name, value, err := SplitProxmoxToken(o.Credentials.ProxmoxMonitorToken)
if err != nil {
return st, fmt.Errorf("proxmoxMonitorToken: %w", err)
}
ip, err := nodeIP(o.Config.Proxmox.APIURL)
if err != nil {
return st, err
}
st.Secrets["PVE_USER"], st.Secrets["PVE_TOKEN_NAME"], st.Secrets["PVE_TOKEN_VALUE"] = user, name, value
// The exporter reaches the node as pve-node, so its certificate's name
// never matches; the node's is self-signed anyway, and the token only
// reads.
st.Config["PVE_VERIFY_SSL"] = "false"
st.Environment = "COMPOSE_PROFILES=pve\nURBEX_PVE_NODE_IP=" + ip + "\n"
return st, nil
}

// SplitProxmoxToken splits an API token, "user@realm!name=secret", into
// the user, the token's name and its secret.
func SplitProxmoxToken(token string) (user, name, value string, err error) {
id, value, ok := strings.Cut(token, "=")
user, name, ok2 := strings.Cut(id, "!")
if !ok || !ok2 || user == "" || name == "" || value == "" || strings.Index(user, "@") < 1 {
return "", "", "", fmt.Errorf("not a Proxmox API token: expected user@realm!name=secret")
}
return user, name, value, nil
}

// nodeIP is the address of the Proxmox API's host, resolved if it is a
// name.
func nodeIP(apiURL string) (string, error) {
u, err := url.Parse(apiURL)
if err != nil || u.Hostname() == "" {
return "", fmt.Errorf("proxmox.apiUrl %q has no host", apiURL)
}
if ip := net.ParseIP(u.Hostname()); ip != nil {
return ip.String(), nil
}
ips, err := net.LookupIP(u.Hostname())
if err != nil || len(ips) == 0 {
return "", fmt.Errorf("resolving the Proxmox node %s: %v", u.Hostname(), err)
}
return ips[0].String(), nil
}

// Keycloak's database and its owner, in the Keycloak Stack's PostgreSQL.
Expand Down Expand Up @@ -208,6 +273,7 @@ func (o *Orchestrator) platformStackDeploys(stacks []PlatformStack) ([]stackDepl
{Path: "secrets.sops.env", Requires: "Redeploy"},
},
ComposeCmdWrapper: ComposeWrapper,
Environment: st.Environment,
}})
}
return deploys, nil
Expand Down
8 changes: 7 additions & 1 deletion internal/cli/doctor.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ func newDoctorCmd() *cobra.Command {
ansible-playbook, sops, age); the workspace (the credentials, the SSH key,
the GitOps repo and its urbex.platform.yaml); with a platform config and a
Proxmox token, the node (the API answers, the node, pool, storage, LXC
template and bridge exist, the token has the privileges urbex needs); and,
template and bridge exist, the token has the privileges urbex needs, and the optional
read-only token for the node's metrics); and,
with Cloudflare in the config, the token and the zone.

It starts with urbex's own version. Each check is ok, warn, fail or skip, with what to do when it isn't ok. It
Expand Down Expand Up @@ -73,6 +74,11 @@ docs/reference/cli.md in urbforge/urbex.`,
p = proxmoxapi.New(cfg.Proxmox.APIURL, creds.ProxmoxToken, cfg.Proxmox.Insecure)
}
doctor.ProxmoxChecks(ctx, p, cfg, r)
var mon doctor.PermissionsReader
if cfg != nil && creds.ProxmoxMonitorToken != "" {
mon = proxmoxapi.New(cfg.Proxmox.APIURL, creds.ProxmoxMonitorToken, cfg.Proxmox.Insecure)
}
doctor.NodeMetricsChecks(ctx, mon, creds.ProxmoxMonitorToken, cfg, r)
var cf doctor.CloudflareReader
if cfg != nil && creds.CloudflareToken != "" {
cf = cloudflare.New(cloudflare.DefaultBaseURL, creds.CloudflareToken, cfg.Cloudflare.AccountID, cfg.Cloudflare.ZoneID)
Expand Down
7 changes: 7 additions & 0 deletions internal/credentials/credentials.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ type Credentials struct {
ProxmoxToken string `yaml:"proxmoxToken,omitempty"`
CloudflareToken string `yaml:"cloudflareToken,omitempty"`
AgeKey string `yaml:"ageKey,omitempty"`
// ProxmoxMonitorToken is a read-only (PVEAuditor) token of the whole
// node, "user@realm!name=secret", for the node's metrics in Grafana
// (urbforge/urbex-cli#94); optional, provided by the user.
ProxmoxMonitorToken string `yaml:"proxmoxMonitorToken,omitempty"`

// Generated by urbex bootstrap.
ForgeAdminPassword string `yaml:"forgeAdminPassword,omitempty"`
Expand Down Expand Up @@ -58,6 +62,7 @@ func (c *Credentials) fields() []struct {
{"URBEX_PROXMOX_TOKEN", &c.ProxmoxToken},
{"URBEX_CLOUDFLARE_TOKEN", &c.CloudflareToken},
{"URBEX_AGE_KEY", &c.AgeKey},
{"URBEX_PROXMOX_MONITOR_TOKEN", &c.ProxmoxMonitorToken},
{"URBEX_FORGE_ADMIN_PASSWORD", &c.ForgeAdminPassword},
{"URBEX_FORGE_TOKEN", &c.ForgeToken},
{"URBEX_KOMODO_ADMIN_PASSWORD", &c.KomodoAdminPassword},
Expand Down Expand Up @@ -309,6 +314,8 @@ var provided = map[string]bool{
"URBEX_PROXMOX_TOKEN": true,
"URBEX_CLOUDFLARE_TOKEN": true,
"URBEX_AGE_KEY": true,
// The node's metrics.
"URBEX_PROXMOX_MONITOR_TOKEN": true,
}

// ClearGenerated forgets every credential urbex bootstrap generated -
Expand Down
62 changes: 62 additions & 0 deletions internal/doctor/nodemetrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package doctor

import (
"context"
"regexp"
"strings"

"github.com/urbforge/urbex-cli/internal/platform"
)

// PermissionsReader is what the node metrics check reads with the
// monitoring token (proxmoxapi.Client).
type PermissionsReader interface {
Permissions(ctx context.Context) (map[string]map[string]bool, error)
}

// nodeMetricsPrivileges are what pve-exporter reads on / : the node's
// status and its storages (PVEAuditor has both).
// proxmoxToken is an API token as urbex stores it: user@realm!name=secret.
var proxmoxToken = regexp.MustCompile(`^[^@!=\s]+@[^@!=\s]+![^@!=\s]+=\S+$`)

var nodeMetricsPrivileges = []string{"Sys.Audit", "Datastore.Audit"}

// NodeMetricsChecks checks the optional read-only token for the Proxmox
// node's metrics (urbforge/urbex-cli#94). m is nil without the token:
// the node metrics are off, which is fine.
func NodeMetricsChecks(ctx context.Context, m PermissionsReader, token string, cfg *platform.Config, r *Report) {
add := func(status, detail, fix string) {
r.add(Check{Group: "proxmox", Name: "node metrics", Status: status, Detail: detail, Fix: fix})
}
if cfg == nil {
return
}
if m == nil {
add(Skip, "off: no proxmoxMonitorToken, so Grafana shows the LXCs but not the node's CPU, memory and storages",
"optional: create a read-only token (docs/reference/credentials.md#proxmox-monitoring-token), 'urbex-op set proxmoxMonitorToken', then 'urbex bootstrap'")
return
}
if !proxmoxToken.MatchString(token) {
add(Fail, "proxmoxMonitorToken isn't a whole token: it needs the token's id before its secret",
"store it again as urbex-monitor@pve!metrics=<secret> ('urbex-op set proxmoxMonitorToken'): the id is what 'pveum user token add' names, the secret its value")
return
}
perms, err := m.Permissions(ctx)
if err != nil {
add(Fail, "the monitoring token doesn't work: "+firstLine(err.Error()),
"check proxmoxMonitorToken (user@realm!name=secret) and that the token exists on the node")
return
}
var missing []string
for _, p := range nodeMetricsPrivileges {
if !perms["/"][p] {
missing = append(missing, p)
}
}
if len(missing) > 0 {
add(Fail, "the monitoring token lacks "+strings.Join(missing, ", ")+" on /",
"on the node, as root: pveum acl modify / --users <its user> --roles PVEAuditor (and --tokens <the token> if it is privilege-separated)")
return
}
add(OK, "read-only token with Sys.Audit and Datastore.Audit on /; 'urbex bootstrap' runs pve-exporter", "")
}
55 changes: 55 additions & 0 deletions internal/doctor/nodemetrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package doctor

import (
"context"
"errors"
"strings"
"testing"

"github.com/urbforge/urbex-cli/internal/platform"
)

type fakePerms struct {
perms map[string]map[string]bool
err error
}

func (f fakePerms) Permissions(context.Context) (map[string]map[string]bool, error) {
return f.perms, f.err
}

func TestNodeMetricsChecks(t *testing.T) {
ctx := context.Background()
cfg := &platform.Config{}
token := "urbex-monitor@pve!metrics=00000000-1111-2222-3333-444444444444"
check := func(m PermissionsReader) Check {
r := &Report{}
NodeMetricsChecks(ctx, m, token, cfg, r)
if len(r.Checks) != 1 {
t.Fatalf("checks: %+v", r.Checks)
}
return r.Checks[0]
}
if c := check(nil); c.Status != Skip || !strings.Contains(c.Detail, "off") || !strings.Contains(c.Fix, "proxmoxMonitorToken") {
t.Errorf("no token: %+v", c)
}
if c := check(fakePerms{perms: map[string]map[string]bool{"/": {"Sys.Audit": true, "Datastore.Audit": true, "VM.Audit": true}}}); c.Status != OK {
t.Errorf("auditor: %+v", c)
}
if c := check(fakePerms{perms: map[string]map[string]bool{"/pool/urbex": {"Sys.Audit": true}}}); c.Status != Fail || !strings.Contains(c.Detail, "Sys.Audit, Datastore.Audit") || !strings.Contains(c.Fix, "PVEAuditor") {
t.Errorf("pool-only token: %+v", c)
}
if c := check(fakePerms{err: errors.New("401 authentication failure")}); c.Status != Fail || !strings.Contains(c.Detail, "401") {
t.Errorf("bad token: %+v", c)
}
// Only the secret, as 'pveum user token add' prints it on its own line.
token = "00000000-1111-2222-3333-444444444444"
if c := check(fakePerms{err: errors.New("401")}); c.Status != Fail || !strings.Contains(c.Detail, "token's id") || !strings.Contains(c.Fix, "urbex-monitor@pve!metrics=") {
t.Errorf("secret only: %+v", c)
}
r := &Report{}
NodeMetricsChecks(ctx, nil, "", nil, r)
if len(r.Checks) != 0 {
t.Error("checked without a config")
}
}
6 changes: 6 additions & 0 deletions internal/platform/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ var Catalog = []Service{
{Label: "prometheus", Port: 9090, Exposure: ExposureAccess},
{Label: "prometheus", Port: 9090, Path: "^/api/v1/(write|otlp)", Exposure: ExposureInternal},
}},
// The Proxmox node's CPU, memory and storages, read from its API with a
// read-only token (urbforge/urbex-cli#94), only when the token is in
// the credentials. Prometheus scrapes it inside the Stack's network:
// it publishes nothing on the LXC, and calls only the node.
{Name: "pve-exporter", Kind: KindPlatform, LXC: "observability", Deploy: DeployGitOps, Provides: []string{"node-metrics"},
OutboundOnly: true},
{Name: "loki", Kind: KindPlatform, LXC: "observability", Deploy: DeployGitOps, Provides: []string{"logs"},
Endpoints: []Endpoint{
{Label: "loki", Port: 3100, Exposure: ExposureAccess},
Expand Down
Loading