kernel_zoo/schemas/benchmark_output.json at main · MetaInfer/kernel_zoo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://kernel_zoo/schemas/benchmark_output.json",
  "title": "benchmark.py stdout output",
  "type": "object",
  "required": ["schema_version", "kernel_id", "ran_at", "runner_id", "arch", "overall", "cases"],
  "additionalProperties": false,
  "properties": {
    "schema_version": {"type": "string", "minLength": 1},
    "kernel_id": {
      "type": "string",
      "pattern": "^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$"
    },
    "ran_at": {"type": "string", "format": "date-time"},
    "runner_id": {"type": "string", "minLength": 1},
    "arch": {"type": "string", "minLength": 1},
    "overall": {
      "type": "object",
      "required": ["accept", "accept_reason", "build_ok"],
      "additionalProperties": false,
      "properties": {
        "accept": {"type": "boolean"},
        "accept_reason": {"type": "string", "minLength": 1},
        "build_ok": {"type": "boolean"}
      }
    },
    "cases": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["test_case_id", "correctness", "accept", "accept_reason", "metrics", "error"],
        "additionalProperties": false,
        "properties": {
          "test_case_id": {
            "type": "string",
            "pattern": "^[A-Za-z0-9._-]+$"
          },
          "correctness": {"enum": ["passed", "failed", "skipped"]},
          "correctness_detail": {
            "type": "object",
            "additionalProperties": false,
            "properties": {
              "max_abs_err": {"type": "number", "minimum": 0},
              "max_rel_err": {"type": "number", "minimum": 0}
            }
          },
          "accept": {"type": "boolean"},
          "accept_reason": {"type": "string", "minLength": 1},
          "metrics": {
            "type": "array",
            "minItems": 1,
            "items": {
              "type": "object",
              "required": ["name", "value"],
              "additionalProperties": false,
              "properties": {
                "name": {"type": "string", "minLength": 1},
                "value": {"type": "number"}
              }
            }
          },
          "error": {"type": ["string", "null"]}
        }
      }
    }
  }
}