feat: support nightly spec tests

packages/beacon-node/package.json

@@ -173,6 +173,7 @@
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
+    "dotenv": "^16.4.5",
     "js-yaml": "^4.1.0",
     "rewiremock": "^3.14.5",
     "rimraf": "^4.4.1",

packages/beacon-node/test/spec/downloadTests.ts

@@ -1,9 +1,27 @@
+import path from "node:path";
+import {config} from "dotenv";
+import {downloadNightlyTests} from "@lodestar/spec-test-util/downloadNightlyTests";
 import {downloadTests} from "@lodestar/spec-test-util/downloadTests";
 import {blsSpecTests, ethereumConsensusSpecsTests} from "./specTestVersioning.js";
 
-for (const downloadTestOpts of [ethereumConsensusSpecsTests, blsSpecTests]) {
-  downloadTests(downloadTestOpts, console.log).catch((e: Error) => {
-    console.error(e);
-    process.exit(1);
-  });
+const [date, repo, branch] = process.argv.slice(2);
+const downloads = [downloadTests(blsSpecTests, console.log)];
+
+if (date) {
+  config({path: path.join(import.meta.dirname, "../../../../.env")});
+
+  const opts = {
+    ...ethereumConsensusSpecsTests,
+    ...(repo && {specTestsRepoUrl: `https://github.com/${repo}`}),
+    ...(branch && {branch}),
+  };
+
+  downloads.push(downloadNightlyTests(opts, console.log, date));
+} else {
+  downloads.push(downloadTests(ethereumConsensusSpecsTests, console.log));
 }
+
+await Promise.all(downloads).catch((e: Error) => {
+  console.error(e);
+  process.exit(1);
+});

packages/beacon-node/test/spec/presets/sanity.test.ts

@@ -23,6 +23,7 @@ const sanity: TestRunnerFn<any, BeaconStateAllForks> = (fork, testName, testSuit
     case "slots":
       return sanitySlots(fork, testName, testSuite);
     case "blocks":
+    case "epoch_boundary":
       return sanityBlocks(fork, testName, testSuite);
     default:
       throw Error(`Unknown sanity test ${testName}`);

packages/spec-test-util/package.json

@@ -19,6 +19,11 @@
       "bun": "./src/downloadTests.ts",
       "types": "./lib/downloadTests.d.ts",
       "import": "./lib/downloadTests.js"
+    },
+    "./downloadNightlyTests": {
+      "bun": "./src/downloadNightlyTests.ts",
+      "types": "./lib/downloadNightlyTests.d.ts",
+      "import": "./lib/downloadNightlyTests.js"
     }
   },
   "files": [

packages/spec-test-util/src/downloadNightlyTests.ts

@@ -0,0 +1,85 @@
+import {fetch} from "@lodestar/utils";
+import {downloadGenericSpecTests} from "./downloadTests.js";
+
+type WorkflowRunsResponse = {workflow_runs: {id: number}[]};
+type ArtifactsListResponse = {artifacts: {archive_download_url: string; expired: boolean; name: string}[]};
+
+async function ghApiFetch<T>(endpoint: string, token: string): Promise<T> {
+  const res = await fetch(`https://api.github.com${endpoint}`, {
+    headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"},
+    signal: AbortSignal.timeout(30_000),
+  });
+
+  if (!res.ok) {
+    throw new Error(
+      res.status === 401 ? "GITHUB_TOKEN is invalid or expired" : `GitHub API ${res.status} (${endpoint})`
+    );
+  }
+
+  return res.json() as Promise<T>;
+}
+
+async function resolveNightlyRunId(repo: string, token: string, date?: string, branch?: string): Promise<number> {
+  const params = new URLSearchParams({status: "success", per_page: "1"});
+  if (branch) params.append("branch", branch);
+  if (date) params.append("created", date);
+
+  const {workflow_runs} = await ghApiFetch<WorkflowRunsResponse>(
+    `/repos/${repo}/actions/workflows/tests.yml/runs?${params}`,
+    token
+  );
+
+  const runId = workflow_runs[0]?.id;
+  if (!runId) {
+    throw new Error(`No successful run found${date ? ` on ${date}` : ""} for ${repo}${branch ? ` (${branch})` : ""}`);
+  }
+  return runId;
+}
+
+export async function downloadNightlyTests(
+  opts: {specTestsRepoUrl: string; outputDir: string; testsToDownload: string[]; branch?: string},
+  log: (msg: string) => void,
+  date?: string
+): Promise<void> {
+  const token = process.env.GITHUB_TOKEN;
+  if (!token) throw new Error("GITHUB_TOKEN is required for nightly downloads");
+
+  const resolvedDate = date === "latest" || !date ? undefined : date;
+  if (resolvedDate && !/^\d{4}-\d{2}-\d{2}$/.test(resolvedDate)) {
+    throw new Error(`Invalid date: "${date}". Expected "latest" or YYYY-MM-DD`);
+  }
+
+  const repo = new URL(opts.specTestsRepoUrl).pathname.slice(1).replace(/\/$/, "");
+  const runId = await resolveNightlyRunId(repo, token, resolvedDate, opts.branch);
+  log(`Resolved nightly${resolvedDate ? ` ${resolvedDate}` : ""} to run ${runId}`);
+
+  const {artifacts} = await ghApiFetch<ArtifactsListResponse>(`/repos/${repo}/actions/runs/${runId}/artifacts`, token);
+
+  const urlByTest: Record<string, string> = {};
+  const available: string[] = [];
+  for (const test of opts.testsToDownload) {
+    const artifact = artifacts.find((a) => a.name === `${test}.tar.gz` && !a.expired);
+    if (artifact) {
+      urlByTest[test] = artifact.archive_download_url;
+      available.push(test);
+    } else {
+      log(`Skipping ${test} (not found in run ${runId})`);
+    }
+  }
+
+  if (available.length === 0) throw new Error(`No matching artifacts found in run ${runId}`);
+
+  const authInit: RequestInit = {headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"}};
+
+  await downloadGenericSpecTests(
+    {
+      specVersion: `nightly-${runId}`,
+      specTestsRepoUrl: opts.specTestsRepoUrl,
+      outputDir: opts.outputDir,
+      testsToDownload: available,
+      testUrls: urlByTest,
+      fetchInit: authInit,
+    },
+    log
+  );
+}

packages/spec-test-util/src/downloadTests.ts

@@ -25,6 +25,8 @@ export interface DownloadGenericTestsOptions<TestNames extends string> {
   outputDir: string;
   specTestsRepoUrl: string;
   testsToDownload: TestNames[];
+  testUrls?: Record<string, string>;
+  fetchInit?: RequestInit;
 }
 
 /**
@@ -39,7 +41,14 @@ export async function downloadTests(opts: DownloadTestsOptions, log: (msg: strin
  * Used by spec tests and SlashingProtectionInterchangeTest
  */
 export async function downloadGenericSpecTests<TestNames extends string>(
-  {specVersion, specTestsRepoUrl, outputDir, testsToDownload}: DownloadGenericTestsOptions<TestNames>,
+  {
+    specVersion,
+    specTestsRepoUrl,
+    outputDir,
+    testsToDownload,
+    testUrls,
+    fetchInit,
+  }: DownloadGenericTestsOptions<TestNames>,
   log: (msg: string) => void = logEmpty
 ): Promise<void> {
   log(`outputDir = ${outputDir}`);
@@ -62,12 +71,13 @@ export async function downloadGenericSpecTests<TestNames extends string>(
 
   await Promise.all(
     testsToDownload.map(async (test) => {
-      const url = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`;
+      const defaultUrl = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`;
       const tarball = path.join(outputDir, `${test}.tar.gz`);
 
       await retry(
         async () => {
-          const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000)});
+          const url = testUrls?.[test] ?? defaultUrl;
+          const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000), ...fetchInit});
 
           if (!res.ok) {
             throw new Error(`Failed to download file from ${url}: ${res.status} ${res.statusText}`);
@@ -95,7 +105,7 @@ export async function downloadGenericSpecTests<TestNames extends string>(
         {
           retries: 3,
           onRetry: (e, attempt) => {
-            log(`Download attempt ${attempt} for ${url} failed: ${e.message}`);
+            log(`Download attempt ${attempt} for ${test} failed: ${e.message}`);
           },
         }
       );