fix: render platforms without paired baselines

src/artifact.js

@@ -308,12 +308,20 @@ function loadArtifacts(root) {
     `no result.json artifacts found under ${root}`,
   );
   const baselines = [];
+  const platformHasBaseline = new Map();
   const results = resultPaths.map((filename) => {
     const result = JSON.parse(fs.readFileSync(filename, "utf8"));
     try {
       validateResult(result, config);
       const baselinePath = path.join(path.dirname(filename), "baseline.json");
-      if (fs.existsSync(baselinePath)) {
+      const hasBaseline = fs.existsSync(baselinePath);
+      const previous = platformHasBaseline.get(result.platform.id);
+      assert(
+        previous === undefined || previous === hasBaseline,
+        `same-runner baseline must be present for every shard of platform ${JSON.stringify(result.platform.id)}`,
+      );
+      platformHasBaseline.set(result.platform.id, hasBaseline);
+      if (hasBaseline) {
         const baseline = validateResult(
           JSON.parse(fs.readFileSync(baselinePath, "utf8")),
           config,
@@ -334,10 +342,6 @@ function loadArtifacts(root) {
       throw new Error(`${filename}: ${error.message}`, { cause: error });
     }
   });
-  assert(
-    baselines.length === 0 || baselines.length === results.length,
-    "same-runner baseline must be present for every result artifact",
-  );
   return {
     config,
     results: mergeShards(results),

src/report.js

@@ -100,11 +100,18 @@ function writeReport(
       ([platformId, result]) =>
         isIndexPaired(result, baseline.platforms?.[platformId], sameRunner),
     );
+    const missingPlatform = Object.keys(current.platforms).some(
+      (platformId) => !baseline.platforms?.[platformId],
+    );
     comparisonNote = `_Compared with [\`${baseline.source.sha.slice(0, 12)}\`](<${baseline.source.url}>) measured in the same runner job${
       paired
         ? "; index-paired signed differences and percentage changes are pairwise medians"
         : ""
-    }._`;
+    }.${
+      missingPlatform
+        ? " Platforms without a paired baseline are marked `new`."
+        : ""
+    }_`;
   } else if (baseline) {
     comparisonNote =
       "_Compared only with the latest matching platform in the main series._";

test/artifact.test.js

@@ -95,7 +95,7 @@ test("validates, writes, loads, and merges distinct shards", () => {
   );
 });
 
-test("requires a same-runner baseline for every artifact", () => {
+test("requires a same-runner baseline for every shard of a platform", () => {
   const config = new Config({ id: "artifact", groups: { core: "^Core" } });
   const root = fs.mkdtempSync(path.join(os.tmpdir(), "benchmark-paired-"));
   writeArtifact(
@@ -111,7 +111,46 @@ test("requires a same-runner baseline for every artifact", () => {
   );
   assert.throws(
     () => loadArtifacts(root),
-    /baseline must be present for every result artifact/u,
+    /baseline must be present for every shard of platform "linux-amd64"/u,
+  );
+});
+
+test("allows a platform without a same-runner baseline", () => {
+  const config = new Config({ id: "artifact", groups: { core: "^Core" } });
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "benchmark-partial-"));
+  writeArtifact(
+    path.join(root, "linux"),
+    config,
+    result("linux", ["BenchmarkCoreLinux"]),
+    result("linux", ["BenchmarkCoreLinux"], {
+      source: {
+        ...result("linux", []).source,
+        sha: baselineSHA,
+        url: `https://github.com/owner/project/commit/${baselineSHA}`,
+      },
+    }),
+  );
+  writeArtifact(
+    path.join(root, "windows"),
+    config,
+    result("windows", ["BenchmarkCoreWindows"], {
+      platform: {
+        id: "windows-amd64",
+        label: "Windows / amd64",
+        os: "windows",
+        arch: "amd64",
+      },
+    }),
+  );
+
+  const loaded = loadArtifacts(root);
+  assert.deepEqual(
+    loaded.results.map((item) => item.platform.id),
+    ["linux-amd64", "windows-amd64"],
+  );
+  assert.deepEqual(
+    loaded.baselines.map((item) => item.platform.id),
+    ["linux-amd64"],
   );
 });
 

test/store-report.test.js

@@ -277,6 +277,46 @@ test("marks values new when no main platform baseline exists", () => {
   assert.match(fs.readFileSync(comment, "utf8"), /\| new \|/u);
 });
 
+test("reports a platform without a paired baseline as new", () => {
+  const config = new Config({
+    id: "history",
+    groups: { core: "^Core" },
+  });
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "benchmark-partial-"));
+  const baseSHA = "3434343434343434343434343434343434343434";
+  const headSHA = "3535353535353535353535353535353535353535";
+  const linux = result(headSHA, 8);
+  const windows = result(headSHA, 12);
+  windows.platform = {
+    id: "windows-amd64",
+    label: "Windows / amd64",
+    os: "windows",
+    arch: "amd64",
+  };
+  const baseline = result(baseSHA, 10);
+  const comment = path.join(root, "comment.md");
+
+  writeReport(
+    comment,
+    "",
+    config,
+    {
+      source: linux.source,
+      platforms: { "linux-amd64": linux, "windows-amd64": windows },
+    },
+    {
+      source: baseline.source,
+      platforms: { "linux-amd64": baseline },
+    },
+    { sameRunner: true },
+  );
+
+  const body = fs.readFileSync(comment, "utf8");
+  assert.match(body, /### Linux \/ amd64[\s\S]*-20\.0% \(better\)/u);
+  assert.match(body, /### Windows \/ amd64[\s\S]*\| new \|/u);
+  assert.match(body, /Platforms without a paired baseline are marked `new`/u);
+});
+
 test("keeps valid history when grouping and chart configuration evolves", () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), "benchmark-config-"));
   const firstConfig = new Config({