diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8ab19d5..24521339 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -233,7 +233,7 @@ jobs: if: github.ref_protected needs: collect runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 permissions: contents: read id-token: write @@ -256,7 +256,7 @@ jobs: name: release-package path: release-artifacts - - name: Publish or verify + - name: Publish validated artifact or verify run: node scripts/publish-or-verify.mjs --package "@openclaw/fs-safe" --artifacts release-artifacts release: diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fb2b21..a51889d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Docs and Tooling + +- Publish only the validated release tarball after asserting its byte identity against the release manifest, and tolerate npm registry propagation with bounded exponential backoff. + ## 0.5.0 - 2026-07-27 ### Highlights diff --git a/scripts/publish-or-verify.mjs b/scripts/publish-or-verify.mjs index 46bb7775..6b8e174f 100644 --- a/scripts/publish-or-verify.mjs +++ b/scripts/publish-or-verify.mjs @@ -1,59 +1,145 @@ import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; -const packageIndex = process.argv.indexOf("--package"); -const artifactsIndex = process.argv.indexOf("--artifacts"); -const packageName = packageIndex >= 0 ? process.argv[packageIndex + 1] : undefined; -const artifactsDir = resolve(artifactsIndex >= 0 ? process.argv[artifactsIndex + 1] : "release-artifacts"); -if (!packageName) throw new Error("--package is required"); +export const REGISTRY_RETRY_DELAYS_MS = [ + 5_000, + 10_000, + 20_000, + 30_000, + 45_000, + 60_000, + 60_000, + 60_000, + 60_000, + 60_000, + 60_000, + 60_000, +]; -const manifest = JSON.parse(readFileSync(join(artifactsDir, "manifest.json"), "utf8")); -const artifact = manifest.find((entry) => entry.name === packageName); -if (!artifact) throw new Error(`release manifest has no entry for ${packageName}`); -const spec = `${artifact.name}@${artifact.version}`; +function sha512Integrity(bytes) { + return `sha512-${createHash("sha512").update(bytes).digest("base64")}`; +} + +export function loadReleaseArtifact(packageName, artifactsDirectory) { + const artifactsDir = resolve(artifactsDirectory); + const manifest = JSON.parse(readFileSync(join(artifactsDir, "manifest.json"), "utf8")); + if (!Array.isArray(manifest)) throw new Error("release manifest must be an array"); + + const artifact = manifest.find((entry) => entry?.name === packageName); + if (!artifact) throw new Error(`release manifest has no entry for ${packageName}`); + if ( + typeof artifact.version !== "string" || + typeof artifact.filename !== "string" || + basename(artifact.filename) !== artifact.filename + ) { + throw new Error(`release manifest has invalid artifact metadata for ${packageName}`); + } + + const artifactPath = resolve(artifactsDir, artifact.filename); + if (dirname(artifactPath) !== artifactsDir) { + throw new Error(`release artifact escapes artifacts directory: ${artifact.filename}`); + } -function registryState() { + const bytes = readFileSync(artifactPath); + const integrity = sha512Integrity(bytes); + if (integrity !== artifact.integrity) { + throw new Error(`${packageName}@${artifact.version} artifact bytes do not match release manifest`); + } + if (Number.isSafeInteger(artifact.size) && bytes.length !== artifact.size) { + throw new Error(`${packageName}@${artifact.version} artifact size does not match release manifest`); + } + + return { ...artifact, path: artifactPath, integrity }; +} + +function registryState(artifact, execNpm) { + const spec = `${artifact.name}@${artifact.version}`; try { - const raw = execFileSync("npm", ["view", spec, "dist", "--json"], { + const raw = execNpm("npm", ["view", spec, "dist", "--json", "--prefer-online"], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], }); const dist = JSON.parse(raw); if (dist.integrity !== artifact.integrity) { - throw new Error(`${spec} exists with different package bytes`); + return { state: "pending", reason: "registry reports different package bytes" }; } if (!dist.attestations?.url) { - throw new Error(`${spec} exists without npm provenance`); + return { state: "pending", reason: "registry has not exposed npm provenance" }; } - return "verified"; + return { state: "verified" }; } catch (error) { const stderr = String(error?.stderr ?? ""); - if (stderr.includes("E404")) return "missing"; - throw error; + if (stderr.includes("E404")) return { state: "missing", reason: "version is not visible" }; + return { state: "pending", reason: error instanceof Error ? error.message : String(error) }; } } -if (registryState() === "verified") { - console.log(`verified ${spec} integrity and provenance`); - process.exit(0); +function sleep(milliseconds) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); } -const published = spawnSync( - "npm", - ["publish", join(artifactsDir, artifact.filename), "--access", "public", "--provenance"], - { stdio: "inherit" }, -); -for (let attempt = 1; attempt <= 12; attempt += 1) { - try { - if (registryState() === "verified") { - console.log(`verified ${spec} integrity and provenance`); - process.exit(0); +export async function publishOrVerify({ + packageName, + artifactsDir, + execNpm = execFileSync, + spawnNpm = spawnSync, + retryDelaysMs = REGISTRY_RETRY_DELAYS_MS, + wait = sleep, + log = console.log, +}) { + if (!packageName) throw new Error("--package is required"); + const artifact = loadReleaseArtifact(packageName, artifactsDir); + const spec = `${artifact.name}@${artifact.version}`; + let publishResult; + + for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) { + const registry = registryState(artifact, execNpm); + if (registry.state === "verified") { + log(`verified ${spec} byte identity and provenance`); + return; } - } catch (error) { - if (attempt === 12) throw error; + + if (registry.state === "missing" && publishResult === undefined) { + publishResult = spawnNpm( + "npm", + ["publish", artifact.path, "--access", "public", "--provenance"], + { stdio: "inherit" }, + ); + if (publishResult.error) { + log(`npm publish could not start: ${publishResult.error.message}`); + } else if (publishResult.status !== 0) { + log(`npm publish exited ${publishResult.status}; checking whether the registry committed it`); + } + } + + if (attempt === retryDelaysMs.length) { + const publishSummary = + publishResult === undefined ? "npm publish was not attempted" : `npm publish exited ${publishResult.status}`; + throw new Error(`${publishSummary}; registry never confirmed ${spec}: ${registry.reason}`); + } + + const delayMs = retryDelaysMs[attempt]; + log( + `registry verification attempt ${attempt + 1}/${retryDelaysMs.length + 1} has not confirmed ${spec}` + + ` (${registry.reason}); retrying in ${delayMs / 1_000}s`, + ); + await wait(delayMs); } - console.log(`registry verification attempt ${attempt}/12 has not confirmed ${spec}`); - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5_000); } -throw new Error(`npm publish exited ${published.status}; registry never confirmed ${spec}`); + +function parseArguments(argv) { + const packageIndex = argv.indexOf("--package"); + const artifactsIndex = argv.indexOf("--artifacts"); + return { + packageName: packageIndex >= 0 ? argv[packageIndex + 1] : undefined, + artifactsDir: resolve(artifactsIndex >= 0 ? argv[artifactsIndex + 1] : "release-artifacts"), + }; +} + +const entryPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : undefined; +if (entryPath === import.meta.url) { + await publishOrVerify(parseArguments(process.argv.slice(2))); +} diff --git a/test/publish-or-verify.test.ts b/test/publish-or-verify.test.ts new file mode 100644 index 00000000..ac4701bf --- /dev/null +++ b/test/publish-or-verify.test.ts @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + REGISTRY_RETRY_DELAYS_MS, + loadReleaseArtifact, + publishOrVerify, +} from "../scripts/publish-or-verify.mjs"; + +const temporaryDirectories: string[] = []; + +async function releaseArtifact(bytes = Buffer.from("validated tarball bytes")) { + const directory = await mkdtemp(join(tmpdir(), "fs-safe-publish-test-")); + temporaryDirectories.push(directory); + const filename = "openclaw-fs-safe-9.9.9.tgz"; + const integrity = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + await writeFile(join(directory, filename), bytes); + await writeFile( + join(directory, "manifest.json"), + `${JSON.stringify([ + { + name: "@openclaw/fs-safe", + version: "9.9.9", + filename, + integrity, + size: bytes.length, + }, + ])}\n`, + ); + return { directory, filename, integrity }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("publish-or-verify", () => { + it("rejects artifact bytes that differ from the validated manifest", async () => { + const artifact = await releaseArtifact(); + await writeFile(join(artifact.directory, artifact.filename), "repacked bytes"); + + expect(() => loadReleaseArtifact("@openclaw/fs-safe", artifact.directory)).toThrow( + "artifact bytes do not match release manifest", + ); + }); + + it("publishes the exact validated tarball path and verifies its registry identity", async () => { + const artifact = await releaseArtifact(); + const registryResponses = [ + Object.assign(new Error("missing"), { stderr: "npm error code E404" }), + JSON.stringify({ integrity: "sha512-stale", attestations: { url: "https://example.test/stale" } }), + JSON.stringify({ integrity: artifact.integrity, attestations: { url: "https://example.test/provenance" } }), + ]; + const execNpm = vi.fn(() => { + const response = registryResponses.shift(); + if (response instanceof Error) throw response; + return response; + }); + const spawnNpm = vi.fn(() => ({ status: 0 })); + const wait = vi.fn(async () => undefined); + + await publishOrVerify({ + packageName: "@openclaw/fs-safe", + artifactsDir: artifact.directory, + execNpm, + spawnNpm, + retryDelaysMs: [5_000, 10_000], + wait, + log: vi.fn(), + }); + + expect(spawnNpm).toHaveBeenCalledOnce(); + expect(spawnNpm).toHaveBeenCalledWith( + "npm", + ["publish", resolve(artifact.directory, artifact.filename), "--access", "public", "--provenance"], + { stdio: "inherit" }, + ); + expect(wait.mock.calls.map(([delay]) => delay)).toEqual([5_000, 10_000]); + }); + + it("backs off through transient mismatches without republishing an existing version", async () => { + const artifact = await releaseArtifact(); + const execNpm = vi + .fn() + .mockReturnValueOnce(JSON.stringify({ integrity: "sha512-stale", attestations: {} })) + .mockReturnValueOnce(JSON.stringify({ integrity: "sha512-stale", attestations: {} })) + .mockReturnValueOnce( + JSON.stringify({ integrity: artifact.integrity, attestations: { url: "https://example.test/provenance" } }), + ); + const spawnNpm = vi.fn(() => ({ status: 0 })); + const wait = vi.fn(async () => undefined); + + await publishOrVerify({ + packageName: "@openclaw/fs-safe", + artifactsDir: artifact.directory, + execNpm, + spawnNpm, + retryDelaysMs: [5_000, 10_000], + wait, + log: vi.fn(), + }); + + expect(spawnNpm).not.toHaveBeenCalled(); + expect(wait.mock.calls.map(([delay]) => delay)).toEqual([5_000, 10_000]); + expect(REGISTRY_RETRY_DELAYS_MS.reduce((total, delay) => total + delay, 0)).toBeGreaterThanOrEqual( + 8 * 60_000, + ); + }); +});