Feature: Gitea, Forgejo and GitLab in the Git widget

api/src/proxy.js

@@ -357,12 +357,17 @@ function fetchJSON(raw, opts = {}) {
               contentType: (res.headers['content-type'] || '').toLowerCase(),
             });
           const body = Buffer.concat(bufs).toString('utf8');
-          if (opts.raw) return done(resolve, { status: res.statusCode, data: body });
+          /* Keep headers opt-in. Some widgets return the reply as is, and headers can carry cookies. */
+          const reply = data =>
+            opts.withHeaders
+              ? { status: res.statusCode, data, headers: res.headers }
+              : { status: res.statusCode, data };
+          if (opts.raw) return done(resolve, reply(body));
           const ct = (res.headers['content-type'] || '').toLowerCase();
           try {
-            done(resolve, { status: res.statusCode, data: JSON.parse(body) });
+            done(resolve, reply(JSON.parse(body)));
           } catch {
-            if (looksLikeMetrics(ct, body)) done(resolve, { status: res.statusCode, data: parsePrometheus(body) });
+            if (looksLikeMetrics(ct, body)) done(resolve, reply(parsePrometheus(body)));
             else if (ct.includes('xml') || body.trimStart().startsWith('<')) {
               const parsed = parseXml(body);
               if (parsed['#truncated']) {
@@ -373,8 +378,8 @@ function fetchJSON(raw, opts = {}) {
                   bytes: body.length,
                 });
               }
-              done(resolve, { status: res.statusCode, data: parsed });
-            } else done(resolve, { status: res.statusCode, data: body });
+              done(resolve, reply(parsed));
+            } else done(resolve, reply(body));
           }
         });
       },

api/src/secret-scope.js

@@ -5,6 +5,7 @@
 
 const { SAVED_ROW, secretSpec, cosmeticSpec, transientSpec } = require('./widget-secrets');
 const { toRows, loginOf } = require('./badge-headers');
+const { withManifestDefaults } = require('./widget-defaults');
 
 function stableEqual(a, b) {
   if (a === b) return true;
@@ -89,7 +90,8 @@ function alignSavedConfig(newConfig, savedConfig, entry) {
 
 function widgetConfigMatchesSaved(newConfig, savedConfig, entry) {
   if (!entry) return false;
-  return stableEqual(stripWidgetSecrets(newConfig, entry), stripWidgetSecrets(savedConfig, entry));
+  const stripped = c => stripWidgetSecrets(withManifestDefaults(c, entry), entry);
+  return stableEqual(stripped(newConfig), stripped(savedConfig));
 }
 
 const blankStored = (obj, old, k) => !!old && old[k] != null && old[k] !== '' && !(obj && obj[k]);

api/src/widget-data.js

@@ -10,6 +10,7 @@ const { preserveWidgetSecrets } = require('./widget-secrets');
 const { alignSavedConfig, leavesStoredSecretBlank, RETYPE_MESSAGE } = require('./secret-scope');
 const { dispatchProvider } = require('./provider-dispatch');
 const { widgetSettings } = require('./widget-settings');
+const { withManifestDefaults } = require('./widget-defaults');
 const { IS_DEMO } = require('./demo');
 const demoData = require('./demo-data');
 const log = require('./log');
@@ -128,7 +129,7 @@ function widgetParams(searchParams, manifest) {
 const KEPT_PER_WIDGET = 16;
 
 async function getWidgetData(item, entry, endpointName, searchParams, fetch, row = null, isOptions = false) {
-  const wc = item.widgetConfig || {};
+  const wc = withManifestDefaults(item.widgetConfig, entry);
   /* An options fetch must run the real path, or the editor shows fabricated
      options. */
   if (IS_DEMO && !isOptions && entry.hasDemoFn) {

api/src/widget-defaults.js

@@ -0,0 +1,58 @@
+/* The secret check and the data function must both read a config through this.
+   If only one does, a stored secret can be restored for one destination and
+   sent to another. */
+
+function defaultsOf(fields) {
+  const seen = new Map();
+  for (const f of Array.isArray(fields) ? fields : []) {
+    if (!f || typeof f.key !== 'string' || !f.key) continue;
+    const usable = f.default !== undefined && f.cosmetic !== true && f.transient !== true && f.type !== 'secret';
+    const value = usable ? JSON.stringify(f.default) : null;
+    const prev = seen.get(f.key);
+    seen.set(f.key, prev === undefined || prev === value ? value : null);
+  }
+  const out = new Map();
+  for (const [k, v] of seen) if (v !== null) out.set(k, v);
+  return out;
+}
+
+function fill(obj, defaults) {
+  let out = obj;
+  for (const [k, v] of defaults) {
+    if (Object.hasOwn(obj, k) && obj[k] !== undefined) continue;
+    if (out === obj) out = { ...obj };
+    out[k] = JSON.parse(v);
+  }
+  return out;
+}
+
+const isObj = v => !!v && typeof v === 'object' && !Array.isArray(v);
+
+function withManifestDefaults(config, entry) {
+  const fields = (entry && entry.manifest && entry.manifest.fields) || [];
+  let out = fill(isObj(config) ? config : {}, defaultsOf(fields));
+  const nested = new Map();
+  for (const f of fields) {
+    if (!f || typeof f.key !== 'string' || !Array.isArray(f.fields)) continue;
+    if (f.type !== 'group' && f.type !== 'object') continue;
+    const prev = nested.get(f.key);
+    nested.set(f.key, { type: f.type, fields: (prev ? prev.fields : []).concat(f.fields) });
+  }
+  for (const [k, { type, fields: sub }] of nested) {
+    const defaults = defaultsOf(sub);
+    if (!defaults.size) continue;
+    const v = out[k];
+    let next = v;
+    if (type === 'group' && Array.isArray(v)) {
+      const rows = v.map(row => (isObj(row) ? fill(row, defaults) : row));
+      if (rows.some((row, i) => row !== v[i])) next = rows;
+    } else if (type === 'object' && isObj(v)) next = fill(v, defaults);
+    if (next !== v) {
+      if (out === config) out = { ...out };
+      out[k] = next;
+    }
+  }
+  return out;
+}
+
+module.exports = { withManifestDefaults };

api/test/github-data.test.js

@@ -49,3 +49,173 @@ test('an empty search is a valid empty list', async () => {
   const r = await dataFn(ctxFor(PRS, { status: 200, data: { total_count: 0, items: [] } }));
   assert.deepEqual(r.items, []);
 });
+
+const { normalizeBase } = require('../src/widget-data');
+const { serviceLink } = require('../../ui/js/service-link.js');
+
+function recordingCtx(config, reply) {
+  const calls = [];
+  const ctx = {
+    config,
+    normalizeBase,
+    serviceLink,
+    fetchJSON: async (url, opts) => {
+      calls.push({ url: new URL(url), opts });
+      return typeof reply === 'function' ? reply(url) : reply;
+    },
+    ...errorParts(),
+  };
+  return { ctx, calls };
+}
+
+const GITEA = { githubService: 'gitea', githubUrl: 'https://git.lan/', githubUser: 'me', githubToken: 't' };
+
+for (const service of ['gitea', 'forgejo']) {
+  test(`${service} pull requests search the token owner's open pull requests`, async () => {
+    const { ctx, calls } = recordingCtx(
+      { ...GITEA, githubService: service, githubPrFilters: ['created', 'review-requested'] },
+      {
+        status: 200,
+        headers: { 'x-total-count': '31' },
+        data: [
+          { number: 4, title: 'Fix', repository: { full_name: 'me/app' }, html_url: 'https://git.lan/me/app/pulls/4' },
+        ],
+      },
+    );
+    const r = await dataFn(ctx);
+    const { url, opts } = calls[0];
+    assert.equal(url.origin + url.pathname, 'https://git.lan/api/v1/repos/issues/search');
+    assert.equal(url.searchParams.get('type'), 'pulls');
+    assert.equal(url.searchParams.get('state'), 'open');
+    assert.equal(url.searchParams.get('created'), 'true');
+    assert.equal(url.searchParams.get('review_requested'), 'true');
+    assert.equal(opts.headers.Authorization, 'token t');
+    assert.equal(r.totalCount, 31);
+    assert.deepEqual(r.items, [{ number: 4, title: 'Fix', repo: 'me/app', url: 'https://git.lan/me/app/pulls/4' }]);
+    assert.equal(r.allUrl, 'https://git.lan/pulls');
+  });
+}
+
+test('gitea counts the listed pull requests when the total header is missing', async () => {
+  const { ctx } = recordingCtx(GITEA, {
+    status: 200,
+    data: [
+      { number: 1, title: 'a' },
+      { number: 2, title: 'b' },
+    ],
+  });
+  assert.equal((await dataFn(ctx)).totalCount, 2);
+});
+
+test('gitea without a server URL fails as a setup problem', async () => {
+  const { ctx } = recordingCtx({ ...GITEA, githubUrl: '' }, { status: 200, data: [] });
+  await assert.rejects(dataFn(ctx), e => e.kind === ctx.KIND.INVALID && /URL/.test(e.message));
+});
+
+for (const [status, kind] of [
+  [401, 'AUTH'],
+  [403, 'AUTH'],
+  [404, 'INVALID'],
+]) {
+  test(`gitea answering ${status} fails as ${kind}`, async () => {
+    const { ctx } = recordingCtx(GITEA, { status, data: { message: 'x' } });
+    await assert.rejects(dataFn(ctx), e => e.kind === ctx.KIND[kind]);
+  });
+}
+
+test('gitea contributions pass the recent heatmap slots on as pairs', async () => {
+  const now = Math.floor(Date.now() / 1000);
+  const { ctx, calls } = recordingCtx(
+    { ...GITEA, githubView: 'contributions' },
+    {
+      status: 200,
+      data: [
+        { timestamp: now - 3600, contributions: 2 },
+        { timestamp: now - 3 * 86400, contributions: '4' },
+        { timestamp: now - 400 * 86400, contributions: 9 },
+        { timestamp: now, contributions: 0 },
+        { timestamp: 'x', contributions: 1 },
+        null,
+      ],
+    },
+  );
+  const r = await dataFn(ctx);
+  assert.equal(calls[0].url.pathname, '/api/v1/users/me/heatmap');
+  assert.equal(r.view, 'contributions');
+  assert.deepEqual(r.slots, [
+    [now - 3600, 2],
+    [now - 3 * 86400, 4],
+  ]);
+  assert.equal(r.profileUrl, 'https://git.lan/me');
+});
+
+test('a server address without a scheme is read and linked over https', async () => {
+  const { ctx, calls } = recordingCtx({ ...GITEA, githubUrl: 'git.lan' }, { status: 200, data: [] });
+  const r = await dataFn(ctx);
+  assert.equal(calls[0].url.origin, 'https://git.lan');
+  assert.equal(r.allUrl, 'https://git.lan/pulls');
+  const plain = recordingCtx({ ...GITEA, githubUrl: 'http://git.lan' }, { status: 200, data: [] });
+  await dataFn(plain.ctx);
+  assert.equal(plain.calls[0].url.origin, 'http://git.lan');
+});
+
+const GITLAB = { githubService: 'gitlab', githubUrl: 'https://gitlab.com', githubUser: 'me', githubToken: 't' };
+
+test('gitlab lists open merge requests by the chosen filters', async () => {
+  const { ctx, calls } = recordingCtx(
+    { ...GITLAB, githubPrFilters: ['assigned', 'mentioned'] },
+    {
+      status: 200,
+      headers: { 'x-total': '12' },
+      data: [
+        {
+          iid: 9,
+          title: 'Bump',
+          references: { full: 'me/app!9' },
+          web_url: 'https://gitlab.com/me/app/-/merge_requests/9',
+        },
+      ],
+    },
+  );
+  const r = await dataFn(ctx);
+  const { url, opts } = calls[0];
+  assert.equal(url.pathname, '/api/v4/merge_requests');
+  assert.equal(url.searchParams.get('state'), 'opened');
+  assert.equal(url.searchParams.get('assignee_username'), 'me');
+  assert.equal(url.searchParams.has('author_username'), false);
+  assert.equal(opts.headers['PRIVATE-TOKEN'], 't');
+  assert.deepEqual(r.filters, ['assigned']);
+  assert.equal(r.totalCount, 12);
+  assert.deepEqual(r.items, [
+    { number: 9, title: 'Bump', repo: 'me/app', url: 'https://gitlab.com/me/app/-/merge_requests/9' },
+  ]);
+  assert.equal(r.allUrl, 'https://gitlab.com/dashboard/merge_requests');
+});
+
+test('gitlab with a saved contributions view fails as a setup problem', async () => {
+  const { ctx, calls } = recordingCtx({ ...GITLAB, githubView: 'contributions' }, { status: 200, data: [] });
+  await assert.rejects(dataFn(ctx), e => e.kind === ctx.KIND.INVALID && /contribution/.test(e.message));
+  assert.equal(calls.length, 0);
+});
+
+test('gitlab with only unsupported filters searches by author', async () => {
+  const { ctx, calls } = recordingCtx({ ...GITLAB, githubPrFilters: ['mentioned'] }, { status: 200, data: [] });
+  assert.deepEqual((await dataFn(ctx)).filters, ['created']);
+  assert.equal(calls[0].url.searchParams.get('author_username'), 'me');
+});
+
+test('an unknown service reads GitHub', async () => {
+  const { ctx, calls } = recordingCtx({ ...PRS, githubService: 'svn' }, { status: 200, data: { items: [] } });
+  await dataFn(ctx);
+  assert.equal(calls[0].url.host, 'api.github.com');
+});
+
+test('github contributions link the profile unless a link URL is set', async () => {
+  const reply = {
+    status: 200,
+    data: { data: { user: { contributionsCollection: { contributionCalendar: { weeks: [] } } } } },
+  };
+  assert.equal((await dataFn(recordingCtx(CONTRIBUTIONS, reply).ctx)).profileUrl, 'https://github.com/octo');
+  const own = { ...CONTRIBUTIONS, githubHref: 'https://me.example' };
+  assert.equal((await dataFn(recordingCtx(own, reply).ctx)).profileUrl, 'https://me.example');
+});

api/test/proxy.test.js

@@ -202,6 +202,23 @@ test('getHostIp returns an empty string when no config file exists', () => {
   assert.equal(getHostIp(), '');
 });
 
+test('fetchJSON returns response headers only when asked', async () => {
+  const server = http.createServer((req, res) => {
+    res.writeHead(200, { 'Content-Type': 'application/json', 'X-Total-Count': '7', 'Set-Cookie': 'sid=1' });
+    res.end('[]');
+  });
+  await new Promise(r => server.listen(0, '127.0.0.1', r));
+  const port = server.address().port;
+  try {
+    const plain = await fetchJSON(`http://127.0.0.1:${port}/`, { timeout: 3000 });
+    assert.deepEqual(Object.keys(plain).sort(), ['data', 'status']);
+    const withHeaders = await fetchJSON(`http://127.0.0.1:${port}/`, { withHeaders: true, timeout: 3000 });
+    assert.equal(withHeaders.headers['x-total-count'], '7');
+  } finally {
+    server.close();
+  }
+});
+
 test('fetchJSON returns the untouched body with opts.raw, and auto-parses without it', async () => {
   const metrics = '# TYPE conduit_connected_clients gauge\nconduit_connected_clients 5\n';
   const server = http.createServer((req, res) => {

api/test/secret-scope.test.js

@@ -229,6 +229,42 @@ test('changing the github view, user or filters keeps the stored token in scope'
   }
 });
 
+test('changing the git service or server URL takes the stored token out of scope', () => {
+  const github = getRegistry().github;
+  const saved = { githubService: 'gitea', githubUrl: 'https://git.a', githubUser: 'me', githubToken: 'STORED' };
+  for (const over of [{ githubService: 'forgejo' }, { githubUrl: 'https://git.b' }]) {
+    assert.equal(widgetConfigMatchesSaved({ ...saved, ...over }, saved, github), false, JSON.stringify(over));
+  }
+});
+
+test('a github widget saved before the service field keeps its token when the editor sends the default', () => {
+  const github = getRegistry().github;
+  const saved = { githubView: 'prs', githubUser: 'me', githubToken: 'STORED' };
+  const posted = { githubService: 'github', githubView: 'prs', githubUser: 'me' };
+  assert.equal(widgetConfigMatchesSaved(posted, saved, github), true);
+  assert.equal(alignSavedConfig(posted, saved, github).config.githubToken, 'STORED');
+  assert.equal(widgetConfigMatchesSaved({ ...posted, githubService: 'gitea' }, saved, github), false);
+});
+
+test('a row or object saved before a defaulted field keeps its secret when the editor sends the default', () => {
+  const saved = {
+    services: [{ id: 'a', url: 'http://vpn', token: 'STORED' }],
+    vpn: { url: 'http://vpn', apiKey: 'K' },
+  };
+  const posted = {
+    services: [{ id: 'a', url: 'http://vpn', type: 'gluetun' }],
+    vpn: { url: 'http://vpn', service: 'gluetun' },
+  };
+  assert.equal(widgetConfigMatchesSaved(posted, saved, connections), true);
+  assert.equal(alignSavedConfig(posted, saved, connections).config.services[0].token, 'STORED');
+  const moved = { ...posted, services: [{ ...posted.services[0], type: 'conduit' }] };
+  assert.equal(alignSavedConfig(moved, saved, connections).config.services[0].token, undefined);
+  assert.equal(
+    widgetConfigMatchesSaved({ ...posted, vpn: { ...posted.vpn, service: 'netbird' } }, saved, connections),
+    false,
+  );
+});
+
 test('a field that reaches the request still takes the stored credential out of scope', () => {
   const r = getRegistry();
   const cases = [