Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions .github/scripts/gitlab-mirror.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {readFileSync, writeFileSync} from 'node:fs';
import {pathToFileURL} from 'node:url';

import {forEachSequential} from './async-iteration.mjs';
import {requestGitLab} from './gitlab-request.mjs';

/** Apply qualified mirror exclusions and omit temporary security-advisory forks. */
export function mirrorRepositories(repos, ignored, rootGroup, sourceOrganization) {
Expand Down Expand Up @@ -41,24 +42,22 @@ export function mirrorTarget(repo, rootGroup, sourceOrganization) {
/** Reconcile namespaces and transfer verified legacy mirrors without deleting or recreating projects. */
export class GitLabMirrors {
constructor({gitlabToken, githubToken, rootGroup = 'lizardbyte', fetchImpl = fetch,
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds))}) {
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)), log = console.error}) {
this.gitlabToken = gitlabToken;
this.githubToken = githubToken;
this.rootGroup = rootGroup;
this.fetch = fetchImpl;
this.sleep = sleep;
this.log = log;
this.groups = new Map();
}

/** Make an authenticated request without following a renamed project's redirect or logging credentials. */
async request(method, path, body) {
const response = await this.fetch(`https://gitlab.com/api/v4${path}`, {
method, redirect: 'manual', signal: AbortSignal.timeout(60000),
headers: {'PRIVATE-TOKEN': this.gitlabToken, 'Content-Type': 'application/json'},
...(body ? {body: JSON.stringify(body)} : {}),
return requestGitLab(method, `/api/v4${path}`, {
token: this.gitlabToken, githubToken: this.githubToken, body,
fetchImpl: this.fetch, sleep: this.sleep, log: this.log,
});
const data = await response.json();
return {status: response.status, data};
}

/** Read the root or create one direct subgroup; never attempt unsupported top-level group creation. */
Expand Down
80 changes: 80 additions & 0 deletions .github/scripts/gitlab-request.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import {readFileSync} from 'node:fs';
import {pathToFileURL} from 'node:url';

const apiOrigin = 'https://gitlab.com';
const apiPrefix = '/api/v4/';
const attempts = 5;
const maximumDelay = 300000;
const pause = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds));

/** Bound diagnostics and remove credentials before writing untrusted API messages to logs. */
function safeDetail(value, credentials) {
let text = typeof value === 'string' ? value : JSON.stringify(value);
for (const credential of credentials.filter(Boolean)) {
for (const representation of [credential, encodeURIComponent(credential)]) {
text = text.replaceAll(representation, '[REDACTED]');
}
}
return text.replaceAll(/https?:\/\/[^\s"'<>]+/g, '[URL]').replaceAll(/\s/g, ' ').slice(0, 400);
}

/** Honor Retry-After seconds or dates, with exponential fallback when GitLab omits the header. */
function retryDelay(header, attempt, now) {
const backoff = 15000 * 2 ** attempt;
if (!header) return backoff;
const serverDelay = /^\d+(?:\.\d+)?$/.test(header)
? Math.ceil(Number(header) * 1000) : Date.parse(header) - now();
return Number.isFinite(serverDelay) ? Math.max(backoff, serverDelay) : backoff;
}

/** Preserve non-JSON error responses while rejecting malformed successful API results. */
function parseResponse(text, status) {
try { return JSON.parse(text); }
catch {
if (status >= 200 && status < 300) throw new Error(`GitLab returned a non-JSON response (HTTP ${status})`);
return {message: text || 'Empty response'};
}
}

/** Share rate-limit handling between namespace preparation and the shell mirror's API calls. */
export async function requestGitLab(method, path, {
token, githubToken, body, fetchImpl = fetch, sleep = pause, now = Date.now, log = console.error,
}, attempt = 0) {
const url = new URL(path, `${apiOrigin}/api/v4/`);
if (!['GET', 'POST', 'PUT'].includes(method) || url.origin !== apiOrigin ||
url.username || url.password || !url.pathname.startsWith(apiPrefix)) {
throw new Error('Invalid GitLab API request target');
}
const response = await fetchImpl(url.href, {
method, redirect: 'manual', signal: AbortSignal.timeout(60000),
headers: {'PRIVATE-TOKEN': token, Accept: 'application/json', 'Content-Type': 'application/json'},
...(body === undefined ? {} : {body: typeof body === 'string' ? body : JSON.stringify(body)}),
});
const text = await response.text();
if (response.status === 429 && attempt < attempts - 1) {
const delay = retryDelay(response.headers.get('retry-after'), attempt, now);
if (delay <= maximumDelay) {
log(`GitLab rate limited the request (HTTP 429); retrying in ${delay / 1000}s (${attempt + 2}/${attempts}).`);
await sleep(delay);
return requestGitLab(method, path, {token, githubToken, body, fetchImpl, sleep, now, log}, attempt + 1);
}
log('GitLab Retry-After exceeds the five-minute automatic wait limit; the request will not be retried.');
}
const data = parseResponse(text, response.status);
const credentials = [token, githubToken];
const requestId = response.headers.get('x-request-id');
const detail = safeDetail(data?.message ?? data?.error ?? 'No error details provided', credentials);
const diagnostic = `${method} ${url.pathname} returned HTTP ${response.status}` +
(requestId ? ` (request ID ${safeDetail(requestId, credentials)})` : '') + `: ${detail}`;
if (response.status >= 400 && !(method === 'GET' && response.status === 404)) log(diagnostic);
return {status: response.status, data, text, diagnostic};
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
if (process.argv.length !== 2) throw new Error('GitLab request helper does not accept CLI arguments');
const payload = readFileSync(0, 'utf8').trim();
const result = await requestGitLab(process.env.GITLAB_REQUEST_METHOD, process.env.GITLAB_REQUEST_URL, {
token: process.env.GITLAB_TOKEN, githubToken: process.env.GITHUB_TOKEN, body: payload || undefined,
});
process.stdout.write(JSON.stringify({status: result.status, body: result.text}));
}
27 changes: 8 additions & 19 deletions .github/scripts/mirror-gitlab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,26 +53,15 @@ if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then
fi

gitlab_request() {
local method="$1"
local url="$2"
local data="${3:-}"
local curl_args=(
--silent
--output "${response_file}"
--write-out '%{http_code}'
--request "${method}"
--header "Accept: application/json"
--header "PRIVATE-TOKEN: ${GITLAB_TOKEN}"
)

if [[ -n "${data}" ]]; then
curl_args+=(
--header "Content-Type: application/json"
--data "${data}"
)
local result
if ! result="$(
GITLAB_REQUEST_METHOD="$1" GITLAB_REQUEST_URL="$2" \
node .github/scripts/gitlab-request.mjs <<< "${3:-}"
)"; then
return 1
fi

curl "${curl_args[@]}" "${url}"
jq -jr '.body' <<< "${result}" > "${response_file}" || return 1
jq -r '.status' <<< "${result}"
}

wait_for_gitlab_import() {
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ and incomplete transfers stop that repository's delivery while other repositorie
verified as private before any transfer or Git push. Existing archived GitLab mirrors require manual handling;
the workflow does not unarchive them. Large-project imports and the existing GitLab SSH transport remain in use.

Namespace preparation and mirror delivery share the same GitLab request helper. HTTP 429 responses are retried
up to five attempts, honoring `Retry-After` with exponential backoff when the header is absent. A requested wait
longer than five minutes is reported instead of retried early. Other HTTP errors, including 404s after project
creation, remain failures and include bounded, credential-redacted API details and the GitLab request ID when
available. No project is recreated to work around those errors.

`GITLAB_TOKEN` needs access to read/create subgroups under `lizardbyte`, create/update projects and their visibility,
transfer existing mirror projects, and use the existing import APIs. Transfers require project Owner access and
permission to create projects in the destination group. See [GitLab project transfers](https://docs.gitlab.com/user/project/working_with_projects/#transfer-a-project).
Expand Down
23 changes: 19 additions & 4 deletions tests/gitlab-mirror.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ function target(owner = 'SunshineStream', name = 'Sunshine', id = 99, visibility
}
function harness(routes) {
const calls = [];
const client = new GitLabMirrors({gitlabToken: 'gitlab-secret', githubToken: 'github-secret', sleep: async () => {},
const client = new GitLabMirrors({gitlabToken: 'gitlab-secret', githubToken: 'github-secret', sleep: async () => {}, log: () => {},
fetchImpl: async (url, options = {}) => {
const key = `${options.method ?? 'GET'} ${url.replace('https://gitlab.com/api/v4', '')}`;
calls.push({key, options, body: options.body ? JSON.parse(options.body) : undefined});
assert.ok(key in routes, `Unexpected request: ${key}`);
const result = typeof routes[key] === 'function' ? routes[key](calls.at(-1)) : routes[key];
if (result instanceof Error) throw result;
const [status, data] = result;
return {status, ok: status >= 200 && status < 300, json: async () => data};
return new Response(JSON.stringify(data), {status});
}});
return {client, calls};
}
Expand Down Expand Up @@ -65,6 +65,21 @@ test('missing subgroups are created under the existing root and cached, retainin
assert.deepEqual(requests[0].body, {parent_id: 1, name: 'SunshineStream', path: 'sunshinestream', visibility: 'public', auto_devops_enabled: false});
});

test('namespace preparation recovers from a plain-text rate limit and continues discovery', async () => {
const run = harness({...base, ...missing});
const fetchImpl = run.client.fetch;
let throttled = false;
run.client.fetch = async (url, options) => {
if (!throttled) {
throttled = true;
return new Response('Rate limit exceeded', {status: 429});
}
return fetchImpl(url, options);
};
assert.equal((await run.client.prepare(target())).namespaceId, 2);
assert.ok(throttled);
});

test('group creation races are reread and inaccessible or incorrect parents fail closed', async () => {
let reads = 0;
const {client} = harness({...base,
Expand Down Expand Up @@ -156,8 +171,8 @@ test('transfer errors and incomplete asynchronous transfers fail instead of crea
test('per-repository preparation failures are saved while other mirrors continue', async () => {
const run = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [403, {}]});
const results = await prepareMirrors([target(), target('LizardByte', 'New', 100)], {
gitlabToken: 'secret', githubToken: 'secret', fetchImpl: async (url, options) => {
if (url.endsWith('lizardbyte%2FNew')) return {status: 404, json: async () => ({})};
gitlabToken: 'secret', githubToken: 'secret', log: () => {}, fetchImpl: async (url, options) => {
if (url.endsWith('lizardbyte%2FNew')) return new Response('{}', {status: 404});
return run.client.fetch(url, options);
},
});
Expand Down
167 changes: 167 additions & 0 deletions tests/gitlab-request.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
import assert from 'node:assert/strict';
import {spawnSync} from 'node:child_process';
import test from 'node:test';

import {requestGitLab} from '../.github/scripts/gitlab-request.mjs';

function harness(responses) {
const calls = [];
const delays = [];
const messages = [];
const options = {
token: 'gitlab-test-token', githubToken: 'github/test-token', now: () => Date.parse('2026-11-09T00:00:00Z'),
sleep: async delay => { delays.push(delay); }, log: message => messages.push(message),
fetchImpl: async (url, init) => {
calls.push({url, ...init});
const response = responses.shift();
assert.ok(response, 'Unexpected extra request');
if (response instanceof Error) throw response;
return response;
},
};
return {calls, delays, messages, options};
}

test('plain-text 429 responses retry before parsing, preserving POST body and authentication', async () => {
const run = harness([
new Response('Rate limit exceeded', {status: 429, headers: {'Retry-After': '90'}}),
new Response('{"id":42}', {status: 201}),
]);
const body = {namespace_id: 2, visibility: 'private'};
const result = await requestGitLab('POST', '/api/v4/projects', {...run.options, body});
assert.equal(result.status, 201);
assert.deepEqual(result.data, {id: 42});
assert.deepEqual(run.delays, [90000]);
assert.equal(run.calls.length, 2);
for (const call of run.calls) {
assert.equal(call.url, 'https://gitlab.com/api/v4/projects');
assert.equal(call.method, 'POST');
assert.equal(call.body, JSON.stringify(body));
assert.equal(call.headers['PRIVATE-TOKEN'], run.options.token);
assert.equal(call.redirect, 'manual');
}
});

test('Retry-After dates and omitted or invalid headers use bounded exponential backoff', async () => {
for (const [header, expected] of [
['Mon, 09 Nov 2026 00:02:00 GMT', 120000], ['', 15000], ['invalid', 15000], ['0', 15000],
]) {
const run = harness([new Response('throttled', {status: 429, headers: {'Retry-After': header}}),
new Response('{}', {status: 200})]);
await requestGitLab('GET', '/api/v4/groups/1', run.options);
assert.deepEqual(run.delays, [expected]);
}
});

test('persistent throttling exhausts five attempts with an HTTP error instead of a JSON error', async () => {
const run = harness(Array.from({length: 5}, () => new Response('Rate limit exceeded', {status: 429})));
const result = await requestGitLab('GET', '/api/v4/projects/42', run.options);
assert.equal(result.status, 429);
assert.deepEqual(run.delays, [15000, 30000, 60000, 120000]);
assert.equal(run.calls.length, 5);
assert.match(result.diagnostic, /HTTP 429: Rate limit exceeded/);
});

test('long server cooldowns are reported without retrying before Retry-After expires', async () => {
const run = harness([new Response('Daily limit exceeded', {status: 429, headers: {'Retry-After': '86400'}})]);
assert.equal((await requestGitLab('POST', '/api/v4/projects', run.options)).status, 429);
assert.equal(run.calls.length, 1);
assert.deepEqual(run.delays, []);
assert.match(run.messages[0], /will not be retried/);
});

test('metadata 404s report GitLab details and request IDs without retrying or recreating projects', async () => {
const run = harness([new Response('{"message":"404 Project Not Found"}', {
status: 404, headers: {'x-request-id': 'request-123'},
})]);
const result = await requestGitLab('PUT', '/api/v4/projects/42', {...run.options, body: {visibility: 'private'}});
assert.equal(result.status, 404);
assert.equal(run.calls.length, 1);
assert.match(run.messages[0], /PUT \/api\/v4\/projects\/42 returned HTTP 404 \(request ID request-123\): 404 Project Not Found/);
assert.deepEqual(run.delays, []);
});

test('expected missing-project lookups remain available to provisioning without error log noise', async () => {
const run = harness([new Response('{"message":"404 Project Not Found"}', {status: 404})]);
assert.equal((await requestGitLab('GET', '/api/v4/projects/new', run.options)).status, 404);
assert.deepEqual(run.messages, []);
});

test('other HTTP errors and ambiguous network failures do not replay mutations', async () => {
for (const status of [401, 403, 409, 500, 503]) {
const run = harness([new Response('Error response', {status})]);
assert.equal((await requestGitLab('POST', '/api/v4/projects', run.options)).status, status);
assert.equal(run.calls.length, 1);
assert.deepEqual(run.delays, []);
}
const run = harness([new Error('Connection reset')]);
await assert.rejects(requestGitLab('POST', '/api/v4/projects', run.options), /Connection reset/);
assert.equal(run.calls.length, 1);
});

test('error diagnostics are bounded and redact tokens, encoded tokens, credential URLs, and newlines', async () => {
const body = 'gitlab-test-token github/test-token github%2Ftest-token\n' +
'https://user:password@example.com/import?token=another-secret ' + 'x'.repeat(1000);
const run = harness([new Response(body, {status: 404})]);
const result = await requestGitLab('PUT', '/api/v4/projects/42', run.options);
assert.equal(result.text, body);
assert.match(run.messages[0], /\[REDACTED\]/);
assert.match(run.messages[0], /\[URL\]/);
assert.doesNotMatch(run.messages[0], /gitlab-test-token|github.test-token|github%2Ftest-token|password|another-secret|\n/);
assert.ok(run.messages[0].length < 500);
});

test('successful non-JSON responses fail clearly without logging their contents', async () => {
const run = harness([new Response('sensitive success body', {status: 200})]);
await assert.rejects(requestGitLab('GET', '/api/v4/projects/42', run.options), /non-JSON response \(HTTP 200\)/);
assert.deepEqual(run.messages, []);
});

test('request validation prevents credentials from reaching other hosts or redirect destinations', async () => {
const run = harness([]);
for (const path of ['https://example.com/api/v4/projects', 'https://user:password@gitlab.com/api/v4/projects',
'/api/v4/../../elsewhere', '/not-api']) {
await assert.rejects(requestGitLab('GET', path, run.options), /Invalid GitLab API request target/);
}
await assert.rejects(requestGitLab('DELETE', '/api/v4/projects/42', run.options), /Invalid GitLab API request target/);
assert.deepEqual(run.calls, []);
});

test('the CLI rejects arguments and invalid API origins before issuing a request', () => {
const path = '.github/scripts/gitlab-request.mjs';
const args = spawnSync(process.execPath, [path, '../output'], {encoding: 'utf8'});
assert.equal(args.status, 1);
assert.match(args.stderr, /does not accept CLI arguments/);
const origin = spawnSync(process.execPath, [path], {
encoding: 'utf8', input: '{}', env: {...process.env, GITLAB_REQUEST_METHOD: 'GET',
GITLAB_REQUEST_URL: 'https://example.com/api/v4/projects', GITLAB_TOKEN: 'test-token'},
});
assert.equal(origin.status, 1);
assert.match(origin.stderr, /Invalid GitLab API request target/);
assert.equal(origin.stdout, '');
assert.doesNotMatch(origin.stderr, /test-token/);
});

test('the shell-facing CLI emits only its response envelope on stdout and redacted diagnostics on stderr', () => {
const code = `
globalThis.fetch = async (url, options) => {
if (url !== 'https://gitlab.com/api/v4/projects/42' || options.body !== '{"visibility":"private"}') {
throw new Error('Request changed in transit');
}
return new Response('{"message":"404 Project Not Found: test-token"}', {
status: 404, headers: {'x-request-id': 'request-cli'},
});
};
process.argv[1] = '.github/scripts/gitlab-request.mjs';
await import('./.github/scripts/gitlab-request.mjs');
`;
const result = spawnSync(process.execPath, ['--input-type=module', '-e', code], {
encoding: 'utf8', input: '{"visibility":"private"}\n', env: {...process.env,
GITLAB_REQUEST_METHOD: 'PUT', GITLAB_REQUEST_URL: 'https://gitlab.com/api/v4/projects/42', GITLAB_TOKEN: 'test-token'},
});
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(JSON.parse(result.stdout), {status: 404, body: '{"message":"404 Project Not Found: test-token"}'});
assert.match(result.stderr, /request ID request-cli/);
assert.match(result.stderr, /404 Project Not Found: \[REDACTED\]/);
assert.doesNotMatch(result.stderr, /test-token/);
});
Loading