Fix: follow a moved Proxmox Backup Server group

api/test/backup-data.test.js

@@ -399,8 +399,10 @@ function pbsTask(ns, groupPath, startS, extra = {}) {
 
 const logPath = t => `/nodes/localhost/tasks/${encodeURIComponent(t.task.upid)}/log`;
 
-function pbsServer({ groups = {}, snaps = {}, runs = [], tasksStatus = 200 }) {
+function pbsServer({ groups = {}, snaps = {}, runs = [], tasksStatus = 200, namespaces = null }) {
   return (p, q) => {
+    if (p === '/admin/datastore/main/namespace' && namespaces)
+      return { status: 200, data: { data: namespaces.map(ns => ({ ns })) } };
     if (p === '/admin/datastore/main/groups') {
       const list = groups[q.get('ns') || ''];
       return list ? { status: 200, data: { data: list } } : { status: 400, data: null };
@@ -550,6 +552,62 @@ test('one failing PBS request marks only its own card', async () => {
   assert.deepEqual(out[2], { error: KIND.UPSTREAM });
 });
 
+test('a PBS group moved to another namespace is followed, and found there on the next poll', async () => {
+  const failed = pbsTask('archive', 'vm/300', NOW_S - 60, { status: 'connection error' });
+  const server = pbsServer({
+    namespaces: ['', 'archive', 'site-b'],
+    groups: { '': [], archive: [group('vm', '300', NOW_S - 3600)], 'site-b': [] },
+    runs: [failed],
+  });
+  const config = { slots: [{ ...PBS_ROW, jobId: 'main::vm/300' }] };
+  const first = pbsCtx(config, 'slots', server);
+  const out = await dataFn(first.ctx);
+  assert.equal(out[0].status, 'error', 'reads the tasks of the namespace it moved to');
+  assert.equal(out[0].id, 'main:archive:vm/300');
+  const second = pbsCtx(config, 'slots', server);
+  await dataFn(second.ctx);
+  const asked = second.calls.map(c => new URL(c.url)).filter(u => u.pathname.endsWith('/groups'));
+  assert.deepEqual(
+    asked.map(u => u.searchParams.get('ns')),
+    ['archive'],
+  );
+  assert.ok(!second.calls.some(c => c.url.includes('/namespace')));
+});
+
+test('a PBS group moved back to where the card was saved is found again', async () => {
+  const config = { slots: [{ ...PBS_ROW, jobId: 'main::vm/304' }] };
+  const away = pbsServer({ namespaces: ['', 'x'], groups: { '': [], x: [group('vm', '304', NOW_S - 60)] } });
+  await dataFn(pbsCtx(config, 'slots', away).ctx);
+  const back = pbsServer({ namespaces: ['', 'x'], groups: { '': [group('vm', '304', NOW_S - 60)], x: [] } });
+  const out = await dataFn(pbsCtx(config, 'slots', back).ctx);
+  assert.equal(out[0].id, 'main::vm/304');
+});
+
+test('a PBS namespace moved away is followed to the namespace that now holds the group', async () => {
+  const server = pbsServer({
+    namespaces: ['', 'site-c'],
+    groups: { '': [], 'site-c': [group('ct', '301', NOW_S - 60)] },
+  });
+  const out = await dataFn(pbsCtx({ slots: [{ ...PBS_ROW, jobId: 'main:site-b:ct/301' }] }, 'slots', server).ctx);
+  assert.equal(out[0].status, 'healthy');
+});
+
+/* Two groups with the same type and id are two machines. Picking one shows the wrong backups. */
+test('a PBS group that now appears in two namespaces is not guessed', async () => {
+  const server = pbsServer({
+    namespaces: ['', 'a', 'b'],
+    groups: { '': [], a: [group('vm', '302', NOW_S - 60)], b: [group('vm', '302', NOW_S - 60)] },
+  });
+  const out = await dataFn(pbsCtx({ slots: [{ ...PBS_ROW, jobId: 'main::vm/302' }] }, 'slots', server).ctx);
+  assert.deepEqual(out, [{ error: KIND.INVALID }]);
+});
+
+test('a PBS group that is gone everywhere stays not found', async () => {
+  const server = pbsServer({ namespaces: ['', 'a'], groups: { '': [], a: [] } });
+  const out = await dataFn(pbsCtx({ slots: [{ ...PBS_ROW, jobId: 'main::vm/303' }] }, 'slots', server).ctx);
+  assert.deepEqual(out, [{ error: KIND.INVALID }]);
+});
+
 test('a PBS reply in the wrong shape marks its card as an upstream error', async () => {
   const { ctx } = pbsCtx({ slots: [{ ...PBS_ROW, jobId: 'main::vm/100' }] }, 'slots', p =>
     p === '/admin/datastore/main/groups' ? { status: 200, data: { data: { 'vm/100': {} } } } : null,

ui/widgets/backup/data.js

@@ -207,6 +207,42 @@ function pbsTaskNs(conn, upid, ctx, mayRead) {
   return p;
 }
 
+/* PBS 4.2 can move a group to another namespace of its datastore. The type and
+   id stay. Kept per card so the search runs once after a move. */
+const _pbsMoved = new Map();
+const PBS_MOVED_MAX = 200;
+
+const pbsSameGroup = ref => g => g['backup-type'] === ref.type && g['backup-id'] === ref.id;
+
+/** The one other namespace of the datastore that holds this group, or null
+    when none does, several do, or a list cannot be read. */
+async function pbsFindMoved(conn, ref, ctx) {
+  let spaces;
+  try {
+    spaces = (await pbsGet(conn, `${pbsStorePath(ref.store)}/namespace`, ctx, PBS_KEEP.ns)).map(n => n.ns || '');
+  } catch {
+    return null;
+  }
+  if (!spaces.length) spaces = [''];
+  if (spaces.length > PBS_MAX_NAMESPACES) return null;
+  const lists = await Promise.all(
+    spaces
+      .filter(ns => ns !== ref.ns)
+      .map(async ns => {
+        const groups = await pbsGet(
+          conn,
+          `${pbsStorePath(ref.store)}/groups${pbsNsQuery(ns)}`,
+          ctx,
+          PBS_KEEP.group,
+        ).catch(() => null);
+        return groups && { ns, group: groups.find(pbsSameGroup(ref)) };
+      }),
+  );
+  if (lists.includes(null)) return null;
+  const hits = lists.filter(l => l.group);
+  return hits.length === 1 ? hits[0] : null;
+}
+
 async function pbsGroupTasks(conn, storeTasks, ref, ctx) {
   if (!storeTasks) return null;
   const workerId = `${ref.store}:${ref.type}/${ref.id}`;
@@ -400,12 +436,19 @@ async function slots(config, ctx) {
     Object.values(pbsInstances).map(async ({ row, slots: gs }) => {
       try {
         const conn = pbsConn(row, ctx);
-        const wanted = gs.map(g => ({ ...g, ref: pbsParseGroupId(g.jobId) }));
+        const wanted = gs.map(g => {
+          const saved = pbsParseGroupId(g.jobId);
+          const movedKey = `${conn.base}|${g.jobId}`;
+          const moved = saved && _pbsMoved.get(movedKey);
+          return { ...g, saved, movedKey, ref: saved && moved != null ? { ...saved, ns: moved } : saved };
+        });
         const spaces = new Map();
         for (const { ref } of wanted) if (ref) spaces.set(`${ref.store}:${ref.ns}`, ref);
         const groupLists = new Map();
         for (const [key, { store, ns }] of spaces) {
-          groupLists.set(key, pbsGet(conn, `${pbsStorePath(store)}/groups${pbsNsQuery(ns)}`, ctx, PBS_KEEP.group));
+          const listed = pbsGet(conn, `${pbsStorePath(store)}/groups${pbsNsQuery(ns)}`, ctx, PBS_KEEP.group);
+          listed.catch(() => {});
+          groupLists.set(key, listed);
         }
         const since = Math.floor(Date.now() / 1000) - PBS_TASK_WINDOW_S;
         const tasksByStore = new Map();
@@ -417,17 +460,33 @@ async function slots(config, ctx) {
           );
         }
         await Promise.all(
-          wanted.map(async ({ i, ref, customName }) => {
+          wanted.map(async ({ i, saved, movedKey, ref: at, customName }) => {
             try {
-              if (!ref) {
+              if (!saved) {
                 result[i] = { error: ctx.KIND.INVALID };
                 return;
               }
-              const groups = await groupLists.get(`${ref.store}:${ref.ns}`);
-              const group = groups.find(g => g['backup-type'] === ref.type && g['backup-id'] === ref.id);
+              let ref = at;
+              let listError = null;
+              let group = null;
+              try {
+                group = (await groupLists.get(`${ref.store}:${ref.ns}`)).find(pbsSameGroup(ref)) || null;
+              } catch (e) {
+                listError = e;
+              }
               if (!group) {
-                result[i] = { error: ctx.KIND.INVALID };
-                return;
+                const moved = await pbsFindMoved(conn, ref, ctx);
+                if (!moved) {
+                  _pbsMoved.delete(movedKey);
+                  if (listError) throw listError;
+                  result[i] = { error: ctx.KIND.INVALID };
+                  return;
+                }
+                ref = { ...saved, ns: moved.ns };
+                group = moved.group;
+                _pbsMoved.delete(movedKey);
+                if (moved.ns !== saved.ns) _pbsMoved.set(movedKey, moved.ns);
+                while (_pbsMoved.size > PBS_MOVED_MAX) _pbsMoved.delete(_pbsMoved.keys().next().value);
               }
               const q = new URLSearchParams({ 'backup-type': ref.type, 'backup-id': ref.id });
               if (ref.ns) q.set('ns', ref.ns);