schema.json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://github.com/JacobPEvans/mlx-benchmarks/schema/v1.json",
  "title": "MLX Benchmark Result",
  "description": "Structured result envelope for a single benchmark suite run. Backwards-compatible v1 — optional fields may be added without bumping schema_version; breaking changes bump the version and change $id.",
  "type": "object",
  "required": [
    "schema_version",
    "timestamp",
    "git_sha",
    "trigger",
    "suite",
    "model",
    "system",
    "results"
  ],
  "additionalProperties": false,
  "properties": {
    "schema_version": {
      "type": "string",
      "const": "1",
      "description": "Schema version — bump when making breaking changes"
    },
    "timestamp": {
      "type": "string",
      "format": "date-time",
      "description": "ISO 8601 UTC timestamp of when the run started"
    },
    "git_sha": {
      "type": "string",
      "minLength": 7,
      "pattern": "^[0-9a-fA-F]{7,64}$",
      "description": "Full or short git commit SHA of the repo at run time"
    },
    "trigger": {
      "type": "string",
      "enum": ["schedule", "pr", "workflow_dispatch", "local"],
      "description": "What triggered this benchmark run"
    },
    "pr_number": {
      "type": ["integer", "null"],
      "description": "Pull request number if trigger is 'pr', otherwise null"
    },
    "suite": {
      "type": "string",
      "enum": [
        "throughput",
        "ttft",
        "tool-calling",
        "code-accuracy",
        "framework-eval",
        "capability-comparison",
        "coding",
        "reasoning",
        "knowledge",
        "evalplus",
        "math-hard"
      ],
      "description": "Which benchmark suite was executed"
    },
    "model": {
      "type": "string",
      "description": "Model name/ID used for inference (e.g. Qwen3.5-122B-A10B-4bit)"
    },
    "model_revision": {
      "type": "string",
      "description": "HuggingFace revision (tag or commit SHA) pinned at inference time"
    },
    "quantization": {
      "type": "string",
      "description": "Quantization scheme as reported by the runtime (e.g. mlx-4bit, mxfp4, mxfp8)"
    },
    "skipped": {
      "type": "boolean",
      "description": "True when the suite was skipped (e.g. MLX server unavailable on standard runner)"
    },
    "seed": {
      "type": "integer",
      "description": "RNG seed used for generation (when applicable)"
    },
    "gen_kwargs": {
      "type": "object",
      "additionalProperties": false,
      "description": "Generation hyperparameters as passed to the inference API",
      "properties": {
        "max_gen_toks": { "type": "integer" },
        "temperature": { "type": "number" },
        "top_p": { "type": "number" },
        "top_k": { "type": "integer" }
      }
    },
    "system": {
      "type": "object",
      "required": ["os", "chip", "memory_gb"],
      "additionalProperties": false,
      "properties": {
        "os": {
          "type": "string",
          "description": "Operating system and version (e.g. macOS 15.3.2)"
        },
        "chip": {
          "type": "string",
          "description": "CPU/chip model (e.g. Apple M4 Max)"
        },
        "memory_gb": {
          "type": "integer",
          "description": "Total system RAM in GB"
        },
        "vllm_mlx_version": {
          "type": "string",
          "description": "vllm-mlx package version if available"
        },
        "runner": {
          "type": "string",
          "description": "GitHub Actions runner label (e.g. macos-latest, self-hosted)"
        },
        "python_version": {
          "type": "string",
          "description": "Python interpreter version (e.g. 3.11.9)"
        },
        "mlx_version": {
          "type": "string",
          "description": "mlx package version"
        },
        "mlx_lm_version": {
          "type": "string",
          "description": "mlx-lm package version"
        },
        "lm_eval_version": {
          "type": "string",
          "description": "lm-eval package version"
        },
        "kernel": {
          "type": "string",
          "description": "OS kernel release (uname -r)"
        }
      }
    },
    "results": {
      "type": "array",
      "description": "Individual test/metric results",
      "items": {
        "type": "object",
        "required": ["name", "metric", "value", "unit"],
        "additionalProperties": false,
        "properties": {
          "name": {
            "type": "string",
            "description": "Test or measurement name (e.g. 'langgraph', 'tok_per_sec_512')"
          },
          "metric": {
            "type": "string",
            "description": "Metric type (e.g. 'latency', 'throughput', 'score', 'f1')"
          },
          "value": {
            "type": "number",
            "description": "Numeric metric value"
          },
          "unit": {
            "type": "string",
            "description": "Unit of measurement (e.g. 'seconds', 'tok/s', 'ratio', 'bool')"
          },
          "duration_seconds": {
            "type": "number",
            "description": "Wall-clock duration attributed to this result (first-class, replacing tags.total_eval_time_s for new runs)"
          },
          "prompt_tokens_per_second": {
            "type": "number",
            "minimum": 0,
            "description": "Aggregate prompt (prefill) throughput for this result: sum of prompt tokens across all samples divided by duration_seconds. Optional — populated when sample-level token counts are recoverable (e.g. lm-eval --log_samples + tokenizer)."
          },
          "decode_tokens_per_second": {
            "type": "number",
            "minimum": 0,
            "description": "Aggregate decode (completion) throughput: sum of response tokens divided by duration_seconds. This is the headline tok/s number end users care about."
          },
          "total_tokens_per_second": {
            "type": "number",
            "minimum": 0,
            "description": "Aggregate end-to-end throughput: (prompt + response) tokens divided by duration_seconds."
          },
          "first_token_latency_ms": {
            "type": "number",
            "minimum": 0,
            "description": "Time to first token in milliseconds for this result (when measurable — typically populated by streaming-aware harnesses, left unset for lm-eval runs)."
          },
          "peak_rss_mb": {
            "type": "number",
            "minimum": 0,
            "description": "Peak resident set size observed during this result, in megabytes. Complements memory_snapshots when per-result granularity is available."
          },
          "tags": {
            "type": "object",
            "description": "Arbitrary string key-value metadata (e.g. framework, output_tokens)",
            "additionalProperties": {
              "type": "string"
            }
          },
          "raw": {
            "description": "Original unmodified output from the underlying tool or script"
          }
        }
      }
    },
    "memory_snapshots": {
      "type": "array",
      "description": "Memory usage measurements at different phases of the run",
      "items": {
        "type": "object",
        "required": ["phase", "rss_gb"],
        "additionalProperties": false,
        "properties": {
          "phase": {
            "type": "string",
            "description": "Run phase (e.g. 'before', 'loading', 'peak', 'after')"
          },
          "rss_gb": {
            "type": "number",
            "description": "Process RSS memory in GB"
          },
          "free_gb": {
            "type": "number",
            "description": "Free system memory in GB"
          },
          "wired_gb": {
            "type": "number",
            "description": "Wired (non-pageable) memory in GB"
          },
          "swap_mb": {
            "type": "number",
            "description": "Swap usage in MB"
          }
        }
      }
    },
    "errors": {
      "type": "array",
      "description": "Non-fatal errors or warnings encountered during the run",
      "items": {
        "type": "string"
      }
    }
  },
  "examples": [
    {
      "schema_version": "1",
      "timestamp": "2026-04-24T18:30:00Z",
      "git_sha": "aaa3ff3",
      "trigger": "local",
      "suite": "reasoning",
      "model": "mlx-community/Qwen3.5-9B-MLX-4bit",
      "gen_kwargs": {
        "max_gen_toks": 4096,
        "temperature": 0.0
      },
      "system": {
        "os": "macOS 26.4.1",
        "chip": "Apple M4 Max",
        "memory_gb": 128,
        "python_version": "3.11.9",
        "lm_eval_version": "0.4.11",
        "kernel": "25.4.0"
      },
      "results": [
        {
          "name": "gsm8k_cot_zeroshot",
          "metric": "exact_match_flexible",
          "value": 0.8,
          "unit": "ratio",
          "duration_seconds": 123.4,
          "tags": {
            "lm_eval_key": "exact_match,flexible-extract",
            "n_limit": "10"
          }
        }
      ],
      "errors": []
    }
  ]
}