runner.mjs

#!/usr/bin/env node
/**
 * Conformance test runner for workspace.json
 *
 * Validates every file in conformance/valid/ against the schema and ensures
 * they pass. Validates every file in conformance/invalid/ against the schema
 * and ensures they fail. Each invalid file must include a top-level _comment
 * field explaining why it is invalid.
 *
 * Exit codes:
 *   0 — all conformance tests pass
 *   1 — one or more tests fail
 *   2 — runner error (missing schema, malformed corpus, etc.)
 */

import { readFileSync, readdirSync, statSync } from "node:fs";
import { join, resolve } from "node:path";
import Ajv2020 from "ajv/dist/2020.js";
import addFormats from "ajv-formats";

const SCHEMA_PATH = resolve("schema/v1.json");
const VALID_DIR = resolve("conformance/valid");
const INVALID_DIR = resolve("conformance/invalid");

let failures = 0;
let totalTests = 0;

function loadSchema() {
  try {
    return JSON.parse(readFileSync(SCHEMA_PATH, "utf-8"));
  } catch (err) {
    console.error(`[ERROR] Cannot load schema at ${SCHEMA_PATH}: ${err.message}`);
    process.exit(2);
  }
}

function listJsonFiles(dir) {
  try {
    return readdirSync(dir)
      .filter((f) => f.endsWith(".json"))
      .map((f) => join(dir, f))
      .filter((p) => statSync(p).isFile());
  } catch (err) {
    console.error(`[ERROR] Cannot read directory ${dir}: ${err.message}`);
    process.exit(2);
  }
}

function loadJson(path) {
  try {
    return JSON.parse(readFileSync(path, "utf-8"));
  } catch (err) {
    return { _parseError: err.message };
  }
}

function testValid(validate, files) {
  console.log(`\n=== Valid corpus (must pass schema) ===\n`);
  for (const file of files) {
    totalTests++;
    const data = loadJson(file);

    if (data._parseError) {
      console.log(`  ❌ ${file}`);
      console.log(`     JSON parse error: ${data._parseError}`);
      failures++;
      continue;
    }

    const valid = validate(data);
    if (valid) {
      console.log(`  ✅ ${file}`);
    } else {
      console.log(`  ❌ ${file}`);
      console.log(`     Schema errors:`);
      for (const error of validate.errors) {
        console.log(`       ${error.instancePath || "/"}: ${error.message}`);
      }
      failures++;
    }
  }
}

function testInvalid(validate, files) {
  console.log(`\n=== Invalid corpus (must fail schema) ===\n`);
  for (const file of files) {
    totalTests++;
    const data = loadJson(file);

    if (data._parseError) {
      console.log(`  ✅ ${file} (parse error counts as invalid)`);
      continue;
    }

    if (!data._comment || typeof data._comment !== "string") {
      console.log(`  ❌ ${file}`);
      console.log(`     Missing required _comment field explaining why this is invalid`);
      failures++;
      continue;
    }

    const valid = validate(data);
    if (valid) {
      console.log(`  ❌ ${file}`);
      console.log(`     Expected schema validation to fail, but it passed.`);
      console.log(`     Reason this should be invalid: ${data._comment}`);
      failures++;
    } else {
      console.log(`  ✅ ${file} (correctly rejected: ${data._comment})`);
    }
  }
}

function main() {
  console.log("workspace.json conformance test runner");
  console.log("=======================================");

  const schema = loadSchema();
  const ajv = new Ajv2020({ allErrors: true, strict: false });
  addFormats(ajv);
  const validate = ajv.compile(schema);

  const validFiles = listJsonFiles(VALID_DIR);
  const invalidFiles = listJsonFiles(INVALID_DIR);

  if (validFiles.length === 0) {
    console.error("[ERROR] No files found in conformance/valid/. Add at least one example.");
    process.exit(2);
  }
  if (invalidFiles.length === 0) {
    console.error("[ERROR] No files found in conformance/invalid/. Add at least one counter-example.");
    process.exit(2);
  }

  testValid(validate, validFiles);
  testInvalid(validate, invalidFiles);

  console.log("\n=======================================");
  console.log(`Results: ${totalTests - failures}/${totalTests} passed`);

  if (failures > 0) {
    console.log(`❌ ${failures} conformance test(s) failed.`);
    process.exit(1);
  }

  console.log("✅ All conformance tests passed.");
  process.exit(0);
}

main();