From 776cc8bafe862077672fbda2ee18ee2b9799182e Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:16:52 -0500 Subject: [PATCH] fix: retry rate-limited GitLab mirror requests --- .github/scripts/gitlab-mirror.mjs | 13 ++- .github/scripts/gitlab-request.mjs | 80 ++++++++++++++ .github/scripts/mirror-gitlab.sh | 27 ++--- .github/workflows/README.md | 6 ++ tests/gitlab-mirror.test.mjs | 23 +++- tests/gitlab-request.test.mjs | 167 +++++++++++++++++++++++++++++ 6 files changed, 286 insertions(+), 30 deletions(-) create mode 100644 .github/scripts/gitlab-request.mjs create mode 100644 tests/gitlab-request.test.mjs diff --git a/.github/scripts/gitlab-mirror.mjs b/.github/scripts/gitlab-mirror.mjs index 2bc35d33..9c22cd84 100644 --- a/.github/scripts/gitlab-mirror.mjs +++ b/.github/scripts/gitlab-mirror.mjs @@ -2,6 +2,7 @@ import {readFileSync, writeFileSync} from 'node:fs'; import {pathToFileURL} from 'node:url'; import {forEachSequential} from './async-iteration.mjs'; +import {requestGitLab} from './gitlab-request.mjs'; /** Apply qualified mirror exclusions and omit temporary security-advisory forks. */ export function mirrorRepositories(repos, ignored, rootGroup, sourceOrganization) { @@ -41,24 +42,22 @@ export function mirrorTarget(repo, rootGroup, sourceOrganization) { /** Reconcile namespaces and transfer verified legacy mirrors without deleting or recreating projects. */ export class GitLabMirrors { constructor({gitlabToken, githubToken, rootGroup = 'lizardbyte', fetchImpl = fetch, - sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds))}) { + sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)), log = console.error}) { this.gitlabToken = gitlabToken; this.githubToken = githubToken; this.rootGroup = rootGroup; this.fetch = fetchImpl; this.sleep = sleep; + this.log = log; this.groups = new Map(); } /** Make an authenticated request without following a renamed project's redirect or logging credentials. */ async request(method, path, body) { - const response = await this.fetch(`https://gitlab.com/api/v4${path}`, { - method, redirect: 'manual', signal: AbortSignal.timeout(60000), - headers: {'PRIVATE-TOKEN': this.gitlabToken, 'Content-Type': 'application/json'}, - ...(body ? {body: JSON.stringify(body)} : {}), + return requestGitLab(method, `/api/v4${path}`, { + token: this.gitlabToken, githubToken: this.githubToken, body, + fetchImpl: this.fetch, sleep: this.sleep, log: this.log, }); - const data = await response.json(); - return {status: response.status, data}; } /** Read the root or create one direct subgroup; never attempt unsupported top-level group creation. */ diff --git a/.github/scripts/gitlab-request.mjs b/.github/scripts/gitlab-request.mjs new file mode 100644 index 00000000..e8141ad1 --- /dev/null +++ b/.github/scripts/gitlab-request.mjs @@ -0,0 +1,80 @@ +import {readFileSync} from 'node:fs'; +import {pathToFileURL} from 'node:url'; + +const apiOrigin = 'https://gitlab.com'; +const apiPrefix = '/api/v4/'; +const attempts = 5; +const maximumDelay = 300000; +const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); + +/** Bound diagnostics and remove credentials before writing untrusted API messages to logs. */ +function safeDetail(value, credentials) { + let text = typeof value === 'string' ? value : JSON.stringify(value); + for (const credential of credentials.filter(Boolean)) { + for (const representation of [credential, encodeURIComponent(credential)]) { + text = text.replaceAll(representation, '[REDACTED]'); + } + } + return text.replaceAll(/https?:\/\/[^\s"'<>]+/g, '[URL]').replaceAll(/\s/g, ' ').slice(0, 400); +} + +/** Honor Retry-After seconds or dates, with exponential fallback when GitLab omits the header. */ +function retryDelay(header, attempt, now) { + const backoff = 15000 * 2 ** attempt; + if (!header) return backoff; + const serverDelay = /^\d+(?:\.\d+)?$/.test(header) + ? Math.ceil(Number(header) * 1000) : Date.parse(header) - now(); + return Number.isFinite(serverDelay) ? Math.max(backoff, serverDelay) : backoff; +} + +/** Preserve non-JSON error responses while rejecting malformed successful API results. */ +function parseResponse(text, status) { + try { return JSON.parse(text); } + catch { + if (status >= 200 && status < 300) throw new Error(`GitLab returned a non-JSON response (HTTP ${status})`); + return {message: text || 'Empty response'}; + } +} + +/** Share rate-limit handling between namespace preparation and the shell mirror's API calls. */ +export async function requestGitLab(method, path, { + token, githubToken, body, fetchImpl = fetch, sleep = pause, now = Date.now, log = console.error, +}, attempt = 0) { + const url = new URL(path, `${apiOrigin}/api/v4/`); + if (!['GET', 'POST', 'PUT'].includes(method) || url.origin !== apiOrigin || + url.username || url.password || !url.pathname.startsWith(apiPrefix)) { + throw new Error('Invalid GitLab API request target'); + } + const response = await fetchImpl(url.href, { + method, redirect: 'manual', signal: AbortSignal.timeout(60000), + headers: {'PRIVATE-TOKEN': token, Accept: 'application/json', 'Content-Type': 'application/json'}, + ...(body === undefined ? {} : {body: typeof body === 'string' ? body : JSON.stringify(body)}), + }); + const text = await response.text(); + if (response.status === 429 && attempt < attempts - 1) { + const delay = retryDelay(response.headers.get('retry-after'), attempt, now); + if (delay <= maximumDelay) { + log(`GitLab rate limited the request (HTTP 429); retrying in ${delay / 1000}s (${attempt + 2}/${attempts}).`); + await sleep(delay); + return requestGitLab(method, path, {token, githubToken, body, fetchImpl, sleep, now, log}, attempt + 1); + } + log('GitLab Retry-After exceeds the five-minute automatic wait limit; the request will not be retried.'); + } + const data = parseResponse(text, response.status); + const credentials = [token, githubToken]; + const requestId = response.headers.get('x-request-id'); + const detail = safeDetail(data?.message ?? data?.error ?? 'No error details provided', credentials); + const diagnostic = `${method} ${url.pathname} returned HTTP ${response.status}` + + (requestId ? ` (request ID ${safeDetail(requestId, credentials)})` : '') + `: ${detail}`; + if (response.status >= 400 && !(method === 'GET' && response.status === 404)) log(diagnostic); + return {status: response.status, data, text, diagnostic}; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.argv.length !== 2) throw new Error('GitLab request helper does not accept CLI arguments'); + const payload = readFileSync(0, 'utf8').trim(); + const result = await requestGitLab(process.env.GITLAB_REQUEST_METHOD, process.env.GITLAB_REQUEST_URL, { + token: process.env.GITLAB_TOKEN, githubToken: process.env.GITHUB_TOKEN, body: payload || undefined, + }); + process.stdout.write(JSON.stringify({status: result.status, body: result.text})); +} diff --git a/.github/scripts/mirror-gitlab.sh b/.github/scripts/mirror-gitlab.sh index 10333d19..2f215ddc 100644 --- a/.github/scripts/mirror-gitlab.sh +++ b/.github/scripts/mirror-gitlab.sh @@ -53,26 +53,15 @@ if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then fi gitlab_request() { - local method="$1" - local url="$2" - local data="${3:-}" - local curl_args=( - --silent - --output "${response_file}" - --write-out '%{http_code}' - --request "${method}" - --header "Accept: application/json" - --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" - ) - - if [[ -n "${data}" ]]; then - curl_args+=( - --header "Content-Type: application/json" - --data "${data}" - ) + local result + if ! result="$( + GITLAB_REQUEST_METHOD="$1" GITLAB_REQUEST_URL="$2" \ + node .github/scripts/gitlab-request.mjs <<< "${3:-}" + )"; then + return 1 fi - - curl "${curl_args[@]}" "${url}" + jq -jr '.body' <<< "${result}" > "${response_file}" || return 1 + jq -r '.status' <<< "${result}" } wait_for_gitlab_import() { diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 04dd8aa8..bd7cf02d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -61,6 +61,12 @@ and incomplete transfers stop that repository's delivery while other repositorie verified as private before any transfer or Git push. Existing archived GitLab mirrors require manual handling; the workflow does not unarchive them. Large-project imports and the existing GitLab SSH transport remain in use. +Namespace preparation and mirror delivery share the same GitLab request helper. HTTP 429 responses are retried +up to five attempts, honoring `Retry-After` with exponential backoff when the header is absent. A requested wait +longer than five minutes is reported instead of retried early. Other HTTP errors, including 404s after project +creation, remain failures and include bounded, credential-redacted API details and the GitLab request ID when +available. No project is recreated to work around those errors. + `GITLAB_TOKEN` needs access to read/create subgroups under `lizardbyte`, create/update projects and their visibility, transfer existing mirror projects, and use the existing import APIs. Transfers require project Owner access and permission to create projects in the destination group. See [GitLab project transfers](https://docs.gitlab.com/user/project/working_with_projects/#transfer-a-project). diff --git a/tests/gitlab-mirror.test.mjs b/tests/gitlab-mirror.test.mjs index 8ec3538c..02325b7d 100644 --- a/tests/gitlab-mirror.test.mjs +++ b/tests/gitlab-mirror.test.mjs @@ -13,7 +13,7 @@ function target(owner = 'SunshineStream', name = 'Sunshine', id = 99, visibility } function harness(routes) { const calls = []; - const client = new GitLabMirrors({gitlabToken: 'gitlab-secret', githubToken: 'github-secret', sleep: async () => {}, + const client = new GitLabMirrors({gitlabToken: 'gitlab-secret', githubToken: 'github-secret', sleep: async () => {}, log: () => {}, fetchImpl: async (url, options = {}) => { const key = `${options.method ?? 'GET'} ${url.replace('https://gitlab.com/api/v4', '')}`; calls.push({key, options, body: options.body ? JSON.parse(options.body) : undefined}); @@ -21,7 +21,7 @@ function harness(routes) { const result = typeof routes[key] === 'function' ? routes[key](calls.at(-1)) : routes[key]; if (result instanceof Error) throw result; const [status, data] = result; - return {status, ok: status >= 200 && status < 300, json: async () => data}; + return new Response(JSON.stringify(data), {status}); }}); return {client, calls}; } @@ -65,6 +65,21 @@ test('missing subgroups are created under the existing root and cached, retainin assert.deepEqual(requests[0].body, {parent_id: 1, name: 'SunshineStream', path: 'sunshinestream', visibility: 'public', auto_devops_enabled: false}); }); +test('namespace preparation recovers from a plain-text rate limit and continues discovery', async () => { + const run = harness({...base, ...missing}); + const fetchImpl = run.client.fetch; + let throttled = false; + run.client.fetch = async (url, options) => { + if (!throttled) { + throttled = true; + return new Response('Rate limit exceeded', {status: 429}); + } + return fetchImpl(url, options); + }; + assert.equal((await run.client.prepare(target())).namespaceId, 2); + assert.ok(throttled); +}); + test('group creation races are reread and inaccessible or incorrect parents fail closed', async () => { let reads = 0; const {client} = harness({...base, @@ -156,8 +171,8 @@ test('transfer errors and incomplete asynchronous transfers fail instead of crea test('per-repository preparation failures are saved while other mirrors continue', async () => { const run = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [403, {}]}); const results = await prepareMirrors([target(), target('LizardByte', 'New', 100)], { - gitlabToken: 'secret', githubToken: 'secret', fetchImpl: async (url, options) => { - if (url.endsWith('lizardbyte%2FNew')) return {status: 404, json: async () => ({})}; + gitlabToken: 'secret', githubToken: 'secret', log: () => {}, fetchImpl: async (url, options) => { + if (url.endsWith('lizardbyte%2FNew')) return new Response('{}', {status: 404}); return run.client.fetch(url, options); }, }); diff --git a/tests/gitlab-request.test.mjs b/tests/gitlab-request.test.mjs new file mode 100644 index 00000000..c45ac4d4 --- /dev/null +++ b/tests/gitlab-request.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import {spawnSync} from 'node:child_process'; +import test from 'node:test'; + +import {requestGitLab} from '../.github/scripts/gitlab-request.mjs'; + +function harness(responses) { + const calls = []; + const delays = []; + const messages = []; + const options = { + token: 'gitlab-test-token', githubToken: 'github/test-token', now: () => Date.parse('2026-11-09T00:00:00Z'), + sleep: async delay => { delays.push(delay); }, log: message => messages.push(message), + fetchImpl: async (url, init) => { + calls.push({url, ...init}); + const response = responses.shift(); + assert.ok(response, 'Unexpected extra request'); + if (response instanceof Error) throw response; + return response; + }, + }; + return {calls, delays, messages, options}; +} + +test('plain-text 429 responses retry before parsing, preserving POST body and authentication', async () => { + const run = harness([ + new Response('Rate limit exceeded', {status: 429, headers: {'Retry-After': '90'}}), + new Response('{"id":42}', {status: 201}), + ]); + const body = {namespace_id: 2, visibility: 'private'}; + const result = await requestGitLab('POST', '/api/v4/projects', {...run.options, body}); + assert.equal(result.status, 201); + assert.deepEqual(result.data, {id: 42}); + assert.deepEqual(run.delays, [90000]); + assert.equal(run.calls.length, 2); + for (const call of run.calls) { + assert.equal(call.url, 'https://gitlab.com/api/v4/projects'); + assert.equal(call.method, 'POST'); + assert.equal(call.body, JSON.stringify(body)); + assert.equal(call.headers['PRIVATE-TOKEN'], run.options.token); + assert.equal(call.redirect, 'manual'); + } +}); + +test('Retry-After dates and omitted or invalid headers use bounded exponential backoff', async () => { + for (const [header, expected] of [ + ['Mon, 09 Nov 2026 00:02:00 GMT', 120000], ['', 15000], ['invalid', 15000], ['0', 15000], + ]) { + const run = harness([new Response('throttled', {status: 429, headers: {'Retry-After': header}}), + new Response('{}', {status: 200})]); + await requestGitLab('GET', '/api/v4/groups/1', run.options); + assert.deepEqual(run.delays, [expected]); + } +}); + +test('persistent throttling exhausts five attempts with an HTTP error instead of a JSON error', async () => { + const run = harness(Array.from({length: 5}, () => new Response('Rate limit exceeded', {status: 429}))); + const result = await requestGitLab('GET', '/api/v4/projects/42', run.options); + assert.equal(result.status, 429); + assert.deepEqual(run.delays, [15000, 30000, 60000, 120000]); + assert.equal(run.calls.length, 5); + assert.match(result.diagnostic, /HTTP 429: Rate limit exceeded/); +}); + +test('long server cooldowns are reported without retrying before Retry-After expires', async () => { + const run = harness([new Response('Daily limit exceeded', {status: 429, headers: {'Retry-After': '86400'}})]); + assert.equal((await requestGitLab('POST', '/api/v4/projects', run.options)).status, 429); + assert.equal(run.calls.length, 1); + assert.deepEqual(run.delays, []); + assert.match(run.messages[0], /will not be retried/); +}); + +test('metadata 404s report GitLab details and request IDs without retrying or recreating projects', async () => { + const run = harness([new Response('{"message":"404 Project Not Found"}', { + status: 404, headers: {'x-request-id': 'request-123'}, + })]); + const result = await requestGitLab('PUT', '/api/v4/projects/42', {...run.options, body: {visibility: 'private'}}); + assert.equal(result.status, 404); + assert.equal(run.calls.length, 1); + assert.match(run.messages[0], /PUT \/api\/v4\/projects\/42 returned HTTP 404 \(request ID request-123\): 404 Project Not Found/); + assert.deepEqual(run.delays, []); +}); + +test('expected missing-project lookups remain available to provisioning without error log noise', async () => { + const run = harness([new Response('{"message":"404 Project Not Found"}', {status: 404})]); + assert.equal((await requestGitLab('GET', '/api/v4/projects/new', run.options)).status, 404); + assert.deepEqual(run.messages, []); +}); + +test('other HTTP errors and ambiguous network failures do not replay mutations', async () => { + for (const status of [401, 403, 409, 500, 503]) { + const run = harness([new Response('Error response', {status})]); + assert.equal((await requestGitLab('POST', '/api/v4/projects', run.options)).status, status); + assert.equal(run.calls.length, 1); + assert.deepEqual(run.delays, []); + } + const run = harness([new Error('Connection reset')]); + await assert.rejects(requestGitLab('POST', '/api/v4/projects', run.options), /Connection reset/); + assert.equal(run.calls.length, 1); +}); + +test('error diagnostics are bounded and redact tokens, encoded tokens, credential URLs, and newlines', async () => { + const body = 'gitlab-test-token github/test-token github%2Ftest-token\n' + + 'https://user:password@example.com/import?token=another-secret ' + 'x'.repeat(1000); + const run = harness([new Response(body, {status: 404})]); + const result = await requestGitLab('PUT', '/api/v4/projects/42', run.options); + assert.equal(result.text, body); + assert.match(run.messages[0], /\[REDACTED\]/); + assert.match(run.messages[0], /\[URL\]/); + assert.doesNotMatch(run.messages[0], /gitlab-test-token|github.test-token|github%2Ftest-token|password|another-secret|\n/); + assert.ok(run.messages[0].length < 500); +}); + +test('successful non-JSON responses fail clearly without logging their contents', async () => { + const run = harness([new Response('sensitive success body', {status: 200})]); + await assert.rejects(requestGitLab('GET', '/api/v4/projects/42', run.options), /non-JSON response \(HTTP 200\)/); + assert.deepEqual(run.messages, []); +}); + +test('request validation prevents credentials from reaching other hosts or redirect destinations', async () => { + const run = harness([]); + for (const path of ['https://example.com/api/v4/projects', 'https://user:password@gitlab.com/api/v4/projects', + '/api/v4/../../elsewhere', '/not-api']) { + await assert.rejects(requestGitLab('GET', path, run.options), /Invalid GitLab API request target/); + } + await assert.rejects(requestGitLab('DELETE', '/api/v4/projects/42', run.options), /Invalid GitLab API request target/); + assert.deepEqual(run.calls, []); +}); + +test('the CLI rejects arguments and invalid API origins before issuing a request', () => { + const path = '.github/scripts/gitlab-request.mjs'; + const args = spawnSync(process.execPath, [path, '../output'], {encoding: 'utf8'}); + assert.equal(args.status, 1); + assert.match(args.stderr, /does not accept CLI arguments/); + const origin = spawnSync(process.execPath, [path], { + encoding: 'utf8', input: '{}', env: {...process.env, GITLAB_REQUEST_METHOD: 'GET', + GITLAB_REQUEST_URL: 'https://example.com/api/v4/projects', GITLAB_TOKEN: 'test-token'}, + }); + assert.equal(origin.status, 1); + assert.match(origin.stderr, /Invalid GitLab API request target/); + assert.equal(origin.stdout, ''); + assert.doesNotMatch(origin.stderr, /test-token/); +}); + +test('the shell-facing CLI emits only its response envelope on stdout and redacted diagnostics on stderr', () => { + const code = ` + globalThis.fetch = async (url, options) => { + if (url !== 'https://gitlab.com/api/v4/projects/42' || options.body !== '{"visibility":"private"}') { + throw new Error('Request changed in transit'); + } + return new Response('{"message":"404 Project Not Found: test-token"}', { + status: 404, headers: {'x-request-id': 'request-cli'}, + }); + }; + process.argv[1] = '.github/scripts/gitlab-request.mjs'; + await import('./.github/scripts/gitlab-request.mjs'); + `; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', code], { + encoding: 'utf8', input: '{"visibility":"private"}\n', env: {...process.env, + GITLAB_REQUEST_METHOD: 'PUT', GITLAB_REQUEST_URL: 'https://gitlab.com/api/v4/projects/42', GITLAB_TOKEN: 'test-token'}, + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), {status: 404, body: '{"message":"404 Project Not Found: test-token"}'}); + assert.match(result.stderr, /request ID request-cli/); + assert.match(result.stderr, /404 Project Not Found: \[REDACTED\]/); + assert.doesNotMatch(result.stderr, /test-token/); +});