Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .github/scripts/async-iteration.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/**
* Process items one at a time, stopping immediately if an operation fails.
* Use this for operations sharing a rate limit, deletion budget, or build state.
*
* @param {Iterable|AsyncIterable} items Items to process lazily.
* @param {(item: any) => Promise<void>} operation Operation to complete before advancing.
* @returns {Promise<void>} Completion of all operations.
*/
export async function forEachSequential(items, operation) {
for await (const item of items) {
await operation(item);
}
}

/**
* Collect pages whose next cursor is only known after loading the current page.
*
* @param {any} initialCursor Cursor for the first page.
* @param {(cursor: any) => Promise<{items: Array, nextCursor: any}>} loadPage Page loader.
* @returns {Promise<Array>} Items in page order. A nullish cursor ends pagination.
*/
export async function collectPages(initialCursor, loadPage) {
let cursor = initialCursor;
const pages = {
[Symbol.asyncIterator]() {
return this;
},
async next() {
if (cursor === null || cursor === undefined) {
return {done: true};
}
const page = await loadPage(cursor);
cursor = page.nextCursor;
return {done: false, value: page.items};
},
};
const results = [];
for await (const items of pages) {
results.push(...items);
}
return results;
}
10 changes: 6 additions & 4 deletions .github/scripts/cleanup-cloudsmith.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import {forEachSequential} from './async-iteration.mjs';

const CLOUDSMITH_API = 'https://api.cloudsmith.io/v1';
const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000;

Expand Down Expand Up @@ -187,14 +189,14 @@ async function loadRepositories(github, organization) {

async function loadReleaseTags(github, organization, repositoryNames) {
const tagsByRepository = new Map();
for (const repository of repositoryNames) {
await forEachSequential(repositoryNames, async (repository) => {
const releases = await github.paginate(github.rest.repos.listReleases, {
owner: organization,
repo: repository,
per_page: 100,
});
tagsByRepository.set(repository.toLowerCase(), publishedReleaseTags(releases));
}
});
return tagsByRepository;
}

Expand Down Expand Up @@ -249,15 +251,15 @@ export async function cleanupCloudsmith({
}
}

for (const {package: packageData} of plan.remove) {
await forEachSequential(plan.remove, async ({package: packageData}) => {
const identifier = packageData.identifier_perm ?? packageData.slug_perm;
if (dryRun) {
core.info(`[dry-run] Delete ${packageLabel(packageData)} (${identifier}).`);
} else {
core.info(`Deleting ${packageLabel(packageData)} (${identifier}).`);
await deleteCloudsmithPackage({owner, repository, token, identifier});
}
}
});

const unknownNames = new Set(
packages
Expand Down
60 changes: 34 additions & 26 deletions .github/scripts/cleanup-containers.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import {collectPages, forEachSequential} from './async-iteration.mjs';

const DOCKER_HUB_API = 'https://hub.docker.com/v2';
const DOCKER_REGISTRY = 'https://registry-1.docker.io';
const GHCR_REGISTRY = 'https://ghcr.io';
Expand Down Expand Up @@ -145,6 +147,17 @@ export function classifyGhcrVersions(versions, releaseTags, {
return result;
}

function* pendingDigests(pending, reachable) {
while (pending.length > 0) {
const digest = pending.pop();
if (!digest || reachable.has(digest)) {
continue;
}
reachable.add(digest);
yield digest;
}
}

/**
* Resolve every child manifest reachable from retained GHCR versions.
*
Expand All @@ -155,19 +168,14 @@ export function classifyGhcrVersions(versions, releaseTags, {
export async function resolveReachableDigests(rootDigests, getManifest) {
const reachable = new Set();
const pending = [...rootDigests];
while (pending.length > 0) {
const digest = pending.pop();
if (!digest || reachable.has(digest)) {
continue;
}
reachable.add(digest);
await forEachSequential(pendingDigests(pending, reachable), async (digest) => {
const manifest = await getManifest(digest);
for (const child of manifest.manifests ?? []) {
if (child.digest && !reachable.has(child.digest)) {
pending.push(child.digest);
}
}
}
});
return reachable;
}

Expand Down Expand Up @@ -213,16 +221,12 @@ async function dockerHubToken(username, accessToken) {
}

async function paginatedHubResults(url, token) {
const results = [];
let next = url;
while (next) {
const page = await fetchJson(next, {
return collectPages(url, async (cursor) => {
const page = await fetchJson(cursor, {
headers: {Authorization: `Bearer ${token}`},
});
results.push(...page.results);
next = page.next;
}
return results;
return {items: page.results, nextCursor: page.next || null};
});
}

async function dockerRegistryToken({namespace, repository, username, accessToken}) {
Expand Down Expand Up @@ -315,11 +319,13 @@ export async function cleanupDockerHub({
let remaining = maxDeletions;
let deferredActions = 0;

for (const hubRepository of hubRepositories.toSorted((a, b) => a.name.localeCompare(b.name))) {
// Keep repository budgets and registry mutations in deterministic order.
const sortedRepositories = hubRepositories.toSorted((a, b) => a.name.localeCompare(b.name));
await forEachSequential(sortedRepositories, async (hubRepository) => {
const githubRepository = githubRepositories.get(hubRepository.name.toLowerCase());
if (!githubRepository) {
core.warning(`Skipping Docker Hub ${namespace}/${hubRepository.name}: no matching GitHub repository.`);
continue;
return;
}

const [tags, releaseTags] = await Promise.all([
Expand All @@ -337,7 +343,7 @@ export async function cleanupDockerHub({
deferredActions += plan.actions.length - selectedActions.length;
let registryToken;

for (const action of selectedActions) {
await forEachSequential(selectedActions, async (action) => {
const description = action.kind === 'manifest'
? `${namespace}/${hubRepository.name}@${action.digest} (${action.tags.length} tags)`
: `${namespace}/${hubRepository.name}:${action.tag}`;
Expand Down Expand Up @@ -368,8 +374,8 @@ export async function cleanupDockerHub({
}
deletedActions += 1;
remaining -= 1;
}
}
});
});

await core.summary
.addHeading(`Docker Hub ${namespace} cleanup`, 2)
Expand Down Expand Up @@ -432,7 +438,8 @@ async function deleteSelectedGhcrVersions({
selectedVersions,
dryRun,
}) {
for (const version of selectedVersions) {
// Tagged parent versions must be deleted before their orphaned children.
await forEachSequential(selectedVersions, async (version) => {
const tags = versionTags(version);
const description = tags.length > 0
? `${namespace}/${packageName}:${tags.join(',')}`
Expand All @@ -448,7 +455,7 @@ async function deleteSelectedGhcrVersions({
versionId: version.id,
});
}
}
});
}

/**
Expand Down Expand Up @@ -485,11 +492,12 @@ export async function cleanupGhcr({
let deferred = 0;
let remaining = maxDeletions;

for (const packageData of packages.sort((a, b) => a.name.localeCompare(b.name))) {
// Finish each package before consuming the shared deletion budget for the next.
await forEachSequential(packages.sort((a, b) => a.name.localeCompare(b.name)), async (packageData) => {
const githubRepository = githubRepositories.get(packageData.name.toLowerCase());
if (!githubRepository) {
core.warning(`Skipping GHCR ${namespace}/${packageData.name}: no matching GitHub repository.`);
continue;
return;
}

const [versions, releaseTags] = await Promise.all([
Expand Down Expand Up @@ -528,7 +536,7 @@ export async function cleanupGhcr({
`Skipping GHCR ${namespace}/${packageData.name}: retained manifest graph could not be read: `
+ error.message,
);
continue;
return;
}

const candidates = ghcrCleanupCandidates(classification, reachable);
Expand All @@ -547,7 +555,7 @@ export async function cleanupGhcr({
});
operations += selectedVersions.length;
remaining -= selectedVersions.length;
}
});

await core.summary
.addHeading(`GHCR ${namespace} cleanup`, 2)
Expand Down
12 changes: 7 additions & 5 deletions .github/scripts/cloudsmith-backfill.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
import path from 'node:path';
import {Readable} from 'node:stream';
import {pipeline} from 'node:stream/promises';
import {forEachSequential} from './async-iteration.mjs';

const SOURCE_OWNER = 'LizardByte';
const CLOUDSMITH_DESTINATION = 'lizardbyte/stable';
Expand Down Expand Up @@ -608,18 +609,19 @@ export async function prepareBackfill({
}

const manifest = [];
for (const release of selected) {
// Package rebuilds and manifest writes share state across assets and releases.
await forEachSequential(selected, async (release) => {
const assets = release.assets.filter((asset) => /\.(?:deb|rpm)$/i.test(asset.name));
if (assets.length === 0) {
core.info(`${release.tag_name}: no DEB or RPM assets`);
continue;
return;
}

core.startGroup(`${release.tag_name}: ${assets.length} package assets`);
try {
const releaseDirectory = path.join(downloads, release.tag_name.replace(/[^0-9A-Za-z_.-]/g, '_'));
mkdirSync(releaseDirectory, {recursive: true});
for (const asset of assets) {
await forEachSequential(assets, async (asset) => {
await stageBackfillAsset({
asset,
release,
Expand All @@ -632,11 +634,11 @@ export async function prepareBackfill({
manifestFile,
core,
});
}
});
} finally {
core.endGroup();
}
}
});

if (manifest.length === 0) {
throw new Error('No stable DEB or RPM assets were staged.');
Expand Down
4 changes: 3 additions & 1 deletion docs/source/_static/js/projects.js
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,9 @@ function initializeProjectsSection() {
createProjectsDescription(section);

const projectList = createProjectsList(section);
fetchAndRenderProjects(projectList);
fetchAndRenderProjects(projectList).catch((error) => {
console.error("Error initializing projects:", error);
});
}

// Initialize when DOM is fully loaded
Expand Down
93 changes: 93 additions & 0 deletions tests/async-iteration.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import assert from 'node:assert/strict';
import {setImmediate} from 'node:timers/promises';
import test from 'node:test';

import {collectPages, forEachSequential} from '../.github/scripts/async-iteration.mjs';

test('forEachSequential waits for each operation before starting the next', async () => {
const gate = Promise.withResolvers();
const events = [];
const completion = forEachSequential([1, 2], async (item) => {
events.push(`start ${item}`);
if (item === 1) {
await gate.promise;
}
events.push(`end ${item}`);
});

await setImmediate();
assert.deepEqual(events, ['start 1']);
gate.resolve();
await completion;
assert.deepEqual(events, ['start 1', 'end 1', 'start 2', 'end 2']);
});

test('forEachSequential stops and closes a lazy iterable on failure', async () => {
const events = [];
function* items() {
try {
events.push('first');
yield 1;
events.push('second');
yield 2;
} finally {
events.push('closed');
}
}
const failure = new Error('operation failed');
await assert.rejects(forEachSequential(items(), async () => {
throw failure;
}), failure);
assert.deepEqual(events, ['first', 'closed']);
});

test('forEachSequential consumes asynchronous iterables', async () => {
async function* items() {
yield 1;
yield 2;
}
const processed = [];
await forEachSequential(items(), async (item) => {
processed.push(item);
});
assert.deepEqual(processed, [1, 2]);
});

test('collectPages follows response cursors and preserves item order', async () => {
const gate = Promise.withResolvers();
const requested = [];
const completion = collectPages('first', async (cursor) => {
requested.push(cursor);
if (cursor === 'first') {
await gate.promise;
return {items: [1, 2], nextCursor: 'last'};
}
return {items: [3], nextCursor: null};
});

await setImmediate();
assert.deepEqual(requested, ['first']);
gate.resolve();
assert.deepEqual(await completion, [1, 2, 3]);
assert.deepEqual(requested, ['first', 'last']);
});

test('collectPages accepts a zero cursor and ends at an undefined cursor', async () => {
assert.deepEqual(await collectPages(0, async (cursor) => {
assert.equal(cursor, 0);
return {items: ['only page']};
}), ['only page']);
});

test('collectPages propagates loader errors without requesting another page', async () => {
const cursors = [];
const failure = new Error('page failed');
await assert.rejects(collectPages(1, async (cursor) => {
cursors.push(cursor);
if (cursor === 2) {
throw failure;
}
return {items: [cursor], nextCursor: cursor + 1};
}), failure);
assert.deepEqual(cursors, [1, 2]);
});
Loading
Loading