Feature: Proxmox health checks and System Summary source

README.md

@@ -41,7 +41,7 @@ Widgets and the services they read:
 - **DNS**: AdGuard, Pi-hole, Technitium, NextDNS
 - **GitHub**: contribution graph and pull requests
 - **Books**: Audiobookshelf, Komga, Kavita
-- **System summary**: CPU, memory, disk, throughput, uptime, and network speed from SpeedTest Tracker or MySpeed. Reports on this machine, or on a host running Glances, Beszel or the Unraid API
+- **System summary**: CPU, memory, disk, throughput, uptime, and network speed from SpeedTest Tracker or MySpeed. Reports on this machine, or on a host running Glances, Beszel, the Unraid API or Proxmox VE
 - **Disk health**: TrueNAS, Scrutiny
 - **Backup**: Duplicati, Kopia
 - **Connections**: Gluetun, Psiphon Conduit, Netbird, Plausible, Umami

api/src/demo-data.js

@@ -73,6 +73,12 @@ function demoHealth(items) {
   return out;
 }
 
+const demoProxmoxGuests = () => [
+  { vmid: 100, name: 'home-assistant', type: 'qemu', node: 'pve' },
+  { vmid: 101, name: 'docker-host', type: 'qemu', node: 'pve' },
+  { vmid: 200, name: 'pihole', type: 'lxc', node: 'pve' },
+];
+
 const helpers = { wave, round };
 
-module.exports = { metrics, helpers, demoBadges, demoHealth, CURVES };
+module.exports = { metrics, helpers, demoBadges, demoHealth, demoProxmoxGuests, CURVES };

api/src/proxmox.js

@@ -0,0 +1,156 @@
+const { fetchUnchecked, pingErrorText } = require('./proxy');
+const { KIND } = require('./api-error');
+const { latest } = require('./latest');
+const { BATCH_MS, PING_MS } = require('./timeouts');
+
+const GUESTS_KEEP_MS = 2 * 60 * 1000;
+
+/* user@realm!tokenid. Anything else in the header value is refused by Proxmox,
+   and a line break would split the request. */
+const TOKEN_ID = /^[^\s@!=]+@[^\s@!=]+![A-Za-z][A-Za-z0-9._-]*$/;
+const SECRET = /^[A-Za-z0-9-]{1,128}$/;
+
+const str = v => (typeof v === 'string' ? v.trim() : '');
+
+/** @param {any} server @returns {{ url: string, tokenId: string, secret: string, skipTlsVerify: boolean } | null} */
+function connectionOf(server) {
+  const p = server?.proxmox;
+  if (!p || typeof p !== 'object') return null;
+  const url = str(p.url)
+    .replace(/[?#].*$/, '')
+    .replace(/\/+$/, '');
+  const tokenId = str(p.tokenId);
+  const secret = str(p.secret);
+  if (!url || !tokenId || !secret) return null;
+  return { url, tokenId, secret, skipTlsVerify: p.skipTlsVerify === true };
+}
+
+/* The message reaches the browser verbatim. Never put upstream text in it. */
+class ProxmoxError extends Error {
+  /** @param {string} message @param {string} kind @param {string} apiCode @param {Record<string, unknown>} [detail] */
+  constructor(message, kind, apiCode, detail) {
+    super(message);
+    this.vouchedMessage = message;
+    this.kind = kind;
+    this.apiCode = apiCode;
+    if (detail) this.detail = detail;
+  }
+}
+
+const badToken = () =>
+  new ProxmoxError('The token ID must look like user@realm!name.', KIND.INVALID, 'invalid.proxmox-token');
+const badUrl = () => new ProxmoxError('That is not a valid URL.', KIND.INVALID, 'invalid.url');
+
+/** @param {{ url: string, tokenId: string, secret: string, skipTlsVerify: boolean }} conn @param {string} path */
+async function pveGet(conn, path, timeout = BATCH_MS) {
+  if (!TOKEN_ID.test(conn.tokenId) || !SECRET.test(conn.secret)) throw badToken();
+  let base;
+  try {
+    base = new URL(conn.url);
+  } catch {
+    throw badUrl();
+  }
+  if (base.protocol !== 'https:' && base.protocol !== 'http:') throw badUrl();
+  const r = await fetchUnchecked(`${conn.url}/api2/json${path}`, {
+    headers: { Authorization: `PVEAPIToken=${conn.tokenId}=${conn.secret}` },
+    timeout,
+    skipTls: conn.skipTlsVerify,
+  });
+  if (r.status === 401) throw new ProxmoxError('Proxmox refused the token.', KIND.UPSTREAM, 'upstream.proxmox-refused');
+  if (r.status === 403)
+    throw new ProxmoxError(
+      'The token is missing a permission. Give it the PVEAuditor role.',
+      KIND.UPSTREAM,
+      'upstream.proxmox-permission',
+    );
+  if (r.status >= 400)
+    throw new ProxmoxError(`Proxmox answered with HTTP ${r.status}.`, KIND.UPSTREAM, 'upstream.status', {
+      status: r.status,
+    });
+  if (!r.data || typeof r.data !== 'object' || !('data' in r.data))
+    throw new ProxmoxError(
+      'Something is listening there, but it is not Proxmox.',
+      KIND.UPSTREAM,
+      'upstream.not-proxmox',
+    );
+  return r.data.data;
+}
+
+/** @returns {Promise<Array<{ vmid: number, name: string, type: string, node: string, status: string }>>} */
+async function listGuests(conn, timeout) {
+  const rows = await pveGet(conn, '/cluster/resources?type=vm', timeout);
+  return (Array.isArray(rows) ? rows : [])
+    .filter(g => g && (g.type === 'qemu' || g.type === 'lxc') && Number.isInteger(g.vmid) && !g.template)
+    .map(g => ({
+      vmid: g.vmid,
+      name: typeof g.name === 'string' && g.name ? g.name : String(g.vmid),
+      type: g.type,
+      node: typeof g.node === 'string' ? g.node : '',
+      status: typeof g.status === 'string' ? g.status : 'unknown',
+    }))
+    .sort((a, b) => a.vmid - b.vmid);
+}
+
+/** @returns {Promise<{ guests: Map<number, { status: string }> } | { error: string, code?: string }>} */
+async function guestStates(conn) {
+  const key = `proxmox:${conn.url}|${conn.tokenId}|${conn.skipTlsVerify}`;
+  const o = await latest(key, () => listGuests(conn, BATCH_MS), { holdMs: BATCH_MS, keepMs: GUESTS_KEEP_MS });
+  if (!o) return { error: 'Proxmox did not answer in time.', code: 'timeout.no-answer' };
+  if (!o.ok)
+    return o.error instanceof ProxmoxError
+      ? { error: o.error.message, code: o.error.apiCode }
+      : { error: pingErrorText(o.error) };
+  return { guests: new Map(o.value.map(g => [g.vmid, { status: g.status }])) };
+}
+
+/* ── stored secret ─────────────────────────────────────────────────────── */
+
+/** @param {any} server */
+function scrubProxmox(server) {
+  const p = server?.proxmox;
+  if (!p || typeof p !== 'object') return;
+  const set = !!str(p.secret);
+  delete p.secret;
+  p.secretSet = set;
+}
+
+/** Refills a blank secret only for the same address, token and certificate
+    setting. Returns true when a stored secret was dropped.
+    @param {any} server @param {any} prevServer @param {boolean} redirected */
+function preserveProxmox(server, prevServer, redirected) {
+  if (!server || typeof server !== 'object') return false;
+  const p = server.proxmox;
+  if (p == null) return false;
+  if (typeof p !== 'object' || Array.isArray(p)) {
+    delete server.proxmox;
+    return false;
+  }
+  const next = { url: str(p.url), tokenId: str(p.tokenId), skipTlsVerify: p.skipTlsVerify === true };
+  const typed = str(p.secret);
+  const old = prevServer?.proxmox && typeof prevServer.proxmox === 'object' ? prevServer.proxmox : null;
+  const oldSecret = str(old?.secret);
+  let dropped = false;
+  if (typed) next.secret = typed;
+  else if (oldSecret) {
+    const same =
+      !redirected &&
+      str(old.url) === next.url &&
+      str(old.tokenId) === next.tokenId &&
+      (old.skipTlsVerify === true) === next.skipTlsVerify;
+    if (same) next.secret = oldSecret;
+    else dropped = true;
+  }
+  server.proxmox = next;
+  return dropped;
+}
+
+module.exports = {
+  connectionOf,
+  pveGet,
+  listGuests,
+  guestStates,
+  scrubProxmox,
+  preserveProxmox,
+  ProxmoxError,
+  TEST_MS: PING_MS,
+};

api/src/routes/config.js

@@ -14,6 +14,7 @@ const backoff = require('../poll-backoff');
 const { stripDisabledCredentials } = require('../auth');
 const { pruneWallpapers } = require('./wallpaper');
 const { normalizeHostList } = require('../../../ui/js/host-names.js');
+const { scrubProxmox, preserveProxmox } = require('../proxmox');
 
 /* Real configs nest under 10 levels. Far deeper input is written to disk and
    then breaks every read of the config. */
@@ -29,6 +30,7 @@ function scrubSecrets(cfg) {
   const safe = structuredClone(cfg);
   scrubAllSecrets(safe);
   if (safe.settings?.background?.apiKey) delete safe.settings.background.apiKey;
+  scrubProxmox(safe.settings?.server);
   if (safe.settings?.auth) {
     delete safe.settings.auth.secret;
     delete safe.settings.auth.passwordHash;
@@ -204,13 +206,15 @@ on('POST', '/api/config', async (req, res) => {
       if (!list) return json(res, 400, { error: 'allowedHosts must be a list of host names', kind: KIND.INVALID });
       server.allowedHosts = list;
     }
+    const proxmoxDropped = preserveProxmox(server, existing.settings?.server, redirected);
     data.settings.server = server;
     /* Migrate first. A migration that renames a widget type after the refill
        stores secrets the new manifest does not scrub. */
     migrate(data);
     const { withheld } = preserveAllSecrets(data, existing);
     if (redirected && existing.settings?.background?.apiKey && !data.settings?.background?.apiKey)
       withheld.push({ id: 'background', label: 'Unsplash' });
+    if (proxmoxDropped) withheld.push({ id: 'proxmox', label: 'Proxmox' });
     ensureSystemItems(data);
     saveConfig(data);
     /* Only now: until the config points at it, the file being replaced is still

api/src/routes/health.js

@@ -19,9 +19,20 @@ const { latest } = require('../latest');
 const { IS_DEMO } = require('../demo');
 const demoData = require('../demo-data');
 const log = require('../log');
+const proxmox = require('../proxmox');
+const { RETYPE_MESSAGE } = require('../secret-scope');
+const { errorBody } = require('../api-error');
 const SOCKET_PROXY_URL_DEFAULT = process.env.SOCKET_PROXY_URL || '';
 const CONTAINERS_KEEP_MS = 2 * 60 * 1000;
 
+/** @param {any} item */
+function guestOf(item) {
+  const hc = item?.monitoring?.healthcheck;
+  if (!hc?.enabled) return null;
+  const id = Number(hc.proxmoxGuest);
+  return Number.isInteger(id) && id > 0 ? id : null;
+}
+
 function socketProxyUrl(cfg) {
   return cfg.settings?.server?.socketProxyUrl || SOCKET_PROXY_URL_DEFAULT;
 }
@@ -139,6 +150,55 @@ on('POST', '/api/docker/test', async (req, res) => {
   }
 });
 
+const MISSING = Object.freeze({
+  error: 'Enter the Proxmox address, token ID and secret.',
+  kind: KIND.INVALID,
+  code: 'invalid.proxmox-missing',
+});
+
+/* A test that leaves the secret blank uses the stored one, but only for the
+   address and token it was stored for. */
+on('POST', '/api/proxmox/test', async (req, res) => {
+  if (!checkOrigin(req, res)) return;
+  const limited = rateLimit(getIp(req), 'proxmox-test', 20, 60_000);
+  if (limited) return json(res, 429, { ok: false, error: limited, kind: KIND.BLOCKED, code: 'blocked.rate-limit' });
+  let body;
+  try {
+    body = JSON.parse(await readBody(req));
+  } catch {
+    return json(res, 400, { ok: false, error: 'url and tokenId required', kind: KIND.INVALID });
+  }
+  const server = {
+    proxmox: { url: body?.url, tokenId: body?.tokenId, secret: body?.secret, skipTlsVerify: body?.skipTlsVerify },
+  };
+  const dropped = proxmox.preserveProxmox(server, loadConfig().settings?.server, false);
+  if (!server.proxmox.secret && dropped)
+    return json(res, 200, { ok: false, error: RETYPE_MESSAGE, kind: KIND.INVALID, code: 'invalid.retype' });
+  const conn = proxmox.connectionOf(server);
+  if (!conn) return json(res, 200, { ok: false, ...MISSING });
+  if (IS_DEMO) return json(res, 200, { ok: true, guests: 0 });
+  try {
+    const guests = await proxmox.listGuests(conn, proxmox.TEST_MS);
+    json(res, 200, { ok: true, guests: guests.length });
+  } catch (e) {
+    json(res, 200, { ok: false, ...errorBody(e) });
+  }
+});
+
+on('GET', '/api/proxmox/guests', async (req, res) => {
+  const limited = rateLimit(getIp(req), 'proxmox-guests', 30, 60_000);
+  if (limited) return json(res, 429, { error: limited, kind: KIND.BLOCKED, code: 'blocked.rate-limit' });
+  if (IS_DEMO) return json(res, 200, { guests: demoData.demoProxmoxGuests() });
+  const conn = proxmox.connectionOf(loadConfig().settings?.server);
+  if (!conn) return json(res, 400, MISSING);
+  try {
+    const guests = await proxmox.listGuests(conn, proxmox.TEST_MS);
+    json(res, 200, { guests: guests.map(({ vmid, name, type, node }) => ({ vmid, name, type, node })) });
+  } catch (e) {
+    json(res, 502, errorBody(e));
+  }
+});
+
 on('GET', '/health', (_, res) => json(res, 200, { ok: true }));
 
 on('GET', '/api/health', async (req, res) => {
@@ -149,6 +209,7 @@ on('GET', '/api/health', async (req, res) => {
     return json(res, 200, demoData.demoHealth(cfg.items));
   }
   const cfg = loadConfig();
+  if (cfg.settings?.server?.healthChecks === false) return json(res, 200, {});
   const socketUrl = socketProxyUrl(cfg);
   /* Without a proxy address every container lookup misses, which would report
      every container-backed app as unhealthy. */
@@ -158,6 +219,9 @@ on('GET', '/api/health', async (req, res) => {
         keepMs: CONTAINERS_KEEP_MS,
       }).then(o => (o?.ok ? o.value : Object.create(null)))
     : Promise.resolve(Object.create(null));
+  const pve = proxmox.connectionOf(cfg.settings?.server);
+  const wantsGuests = cfg.items.some(i => i.type === 'app' && guestOf(i) != null);
+  const guestsReady = pve && wantsGuests ? proxmox.guestStates(pve) : null;
   const result = Object.create(null);
   await Promise.allSettled(
     cfg.items
@@ -166,6 +230,7 @@ on('GET', '/api/health', async (req, res) => {
         const mon = item.monitoring?.healthcheck || {};
         const cName = socketUrl ? mon.container || item.container || '' : '';
         const ping = mon.pingUrl || item.ping || '';
+        const vmid = guestOf(item);
         /* Build the entry up rather than replace it. An item can have both a
            container and a ping, and the tile needs the detail from each. */
         let unhealthy = false;
@@ -193,7 +258,21 @@ on('GET', '/api/health', async (req, res) => {
           detail.pingStatus = r.status;
           detail.pingError = r.error;
         }
-        if (cName || ping) result[item.id] = { unhealthy, ...detail };
+        if (vmid != null) {
+          const r = guestsReady
+            ? await guestsReady
+            : { error: 'Proxmox is not set up in Settings.', code: 'invalid.proxmox-missing' };
+          if ('error' in r) {
+            unhealthy = true;
+            detail.proxmoxError = r.error;
+            if (r.code) detail.proxmoxCode = r.code;
+          } else {
+            const g = r.guests.get(vmid);
+            detail.guestState = g ? g.status : 'missing';
+            if (detail.guestState !== 'running') unhealthy = true;
+          }
+        }
+        if (cName || ping || vmid != null) result[item.id] = { unhealthy, ...detail };
       }),
   );
   json(res, 200, result);