diff --git a/ScratchV-topic06-deliverable/LICENSE b/ScratchV-topic06-deliverable/LICENSE new file mode 100644 index 0000000..87324d7 --- /dev/null +++ b/ScratchV-topic06-deliverable/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 ScratchV + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ScratchV-topic06-deliverable/docs/topic06_bench_suite_usage.md b/ScratchV-topic06-deliverable/docs/topic06_bench_suite_usage.md new file mode 100644 index 0000000..ea752b3 --- /dev/null +++ b/ScratchV-topic06-deliverable/docs/topic06_bench_suite_usage.md @@ -0,0 +1,166 @@ +# 课题 6:ScratchV 课程版性能测试套件使用说明 + +本文档只说明当前作业交付使用的课程版测试套件:`run_tests.py` 和 `tests_main/`。 + +## 1. 目录结构 + +```text +run_tests.py # 自动化测试脚本 + +tests_main/ + activation/ # relu 等激活函数用例 + branch/ # if/else 分支用例 + elementwise/ # add 和链式 add 用例 + loop/ # for/endfor 循环用例 + reduction/ # dot/reduction 用例 + tensor/ # matmul/tensor 用例 + +build/ # 编译后生成的 RISC-V 汇编 + +reports/ + report.md # Markdown 测试报告 + report.html # HTML 测试报告 + course_report_instructions.png + benchmark_baseline.json + +.github/workflows/ + benchmark.yml # CI 示例,运行课程版测试套件 +``` + +当前 `tests_main/` 下有 23 个 DSL 用例,覆盖算术、神经网络算子、循环、if/else 分支、矩阵计算和组合场景。 + +## 2. 运行测试 + +在项目根目录运行: + +```powershell +python run_tests.py +``` + +运行后会自动: + +- 遍历 `tests_main/` 下的 `.dsl` 文件。 +- 调用 ScratchV 编译器生成汇编。 +- 调用 TinyFive 适配器模拟执行。 +- 使用参考执行器计算实际返回值。 +- 对比实际返回值和 `.meta.json` 中的预期返回值。 +- 统计 PASS/FAIL 和指令数。 +- 生成报告。 + +## 3. Benchmark 模式 + +重复运行 3 次并取平均: + +```powershell +python run_tests.py --benchmark 3 +``` + +报告会记录: + +- 平均指令数 +- 最小指令数 +- 最大指令数 +- 95% 置信区间 +- 基线指令数 +- 性能变化率 +- 是否性能退化 + +## 4. 性能基线和退化判断 + +第一次生成基线: + +```powershell +python run_tests.py --benchmark 3 --update-baseline +``` + +之后正常运行: + +```powershell +python run_tests.py --benchmark 3 +``` + +判断规则: + +- 当前平均指令数比基线高出 5% 以上,判定为性能退化。 +- 低于或等于 5% 的波动不算退化。 +- 基线文件保存在 `reports/benchmark_baseline.json`。 + +## 5. 报告文件 + +运行后生成: + +```text +reports/report.md +reports/report.html +reports/course_report_instructions.png +``` + +`report.md` 适合提交作业或放进文档。`report.html` 适合演示,包含表格和 `matplotlib` 生成的性能图表。 + +## 6. 添加新测试用例 + +每个课程版用例由两个文件组成: + +```text +tests_main/{category}/{name}.dsl +tests_main/{category}/{name}.meta.json +``` + +`.dsl` 示例: + +```text +# Simple add +result = add(a, b) +return result +``` + +`.meta.json` 示例: + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +添加后运行: + +```powershell +python run_tests.py --benchmark 3 +``` + +如果失败,优先检查 DSL 语法、输入变量名、预期输出和当前编译器是否支持该算子。 + +## 7. if/else 分支语法 + +当前支持简单分支: + +```text +if flag +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif +``` + +规则: + +- `flag` 非 0 时走 `if` 分支。 +- `flag` 为 0 时走 `else` 分支。 +- 当前不支持复杂比较表达式,比如 `if a > b`。 + +## 8. CI + +`.github/workflows/benchmark.yml` 会在 push 和 pull request 时运行: + +- `python -m pytest -q` +- `python run_tests.py --benchmark 3` + +CI 会上传课程版报告文件,方便查看每次修改后的正确性和性能变化。 diff --git a/ScratchV-topic06-deliverable/pyproject.toml b/ScratchV-topic06-deliverable/pyproject.toml new file mode 100644 index 0000000..6d3be8a --- /dev/null +++ b/ScratchV-topic06-deliverable/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "scratchv" +version = "0.1.0" +description = "A compiler from ONNX models to RISC-V assembly" +requires-python = ">=3.10" +dependencies = [ + "onnx>=1.14", + "numpy>=1.24", + "protobuf>=4.21", + "jinja2>=3.1", + "matplotlib>=3.7", +] + +[project.scripts] +scratchv = "scratchv.main:main" + +[tool.setuptools.packages.find] +include = ["scratchv*"] diff --git a/ScratchV-topic06-deliverable/reports/benchmark_baseline.json b/ScratchV-topic06-deliverable/reports/benchmark_baseline.json new file mode 100644 index 0000000..2f42a38 --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/benchmark_baseline.json @@ -0,0 +1,117 @@ +{ + "add_relu_relu": { + "category": "activation", + "avg_instr_count": 9.0, + "runs": 3 + }, + "relu_add": { + "category": "activation", + "avg_instr_count": 8.0, + "runs": 3 + }, + "relu_only": { + "category": "activation", + "avg_instr_count": 7.0, + "runs": 3 + }, + "relu_twice": { + "category": "activation", + "avg_instr_count": 8.0, + "runs": 3 + }, + "if_else": { + "category": "branch", + "avg_instr_count": 26.0, + "runs": 3 + }, + "if_relu": { + "category": "branch", + "avg_instr_count": 22.0, + "runs": 3 + }, + "if_then": { + "category": "branch", + "avg_instr_count": 26.0, + "runs": 3 + }, + "add_chain": { + "category": "elementwise", + "avg_instr_count": 8.0, + "runs": 3 + }, + "add_chain_3": { + "category": "elementwise", + "avg_instr_count": 9.0, + "runs": 3 + }, + "add_fan_in_4": { + "category": "elementwise", + "avg_instr_count": 9.0, + "runs": 3 + }, + "add_reuse": { + "category": "elementwise", + "avg_instr_count": 8.0, + "runs": 3 + }, + "vector_add": { + "category": "elementwise", + "avg_instr_count": 7.0, + "runs": 3 + }, + "loop_add_4": { + "category": "loop", + "avg_instr_count": 27.0, + "runs": 3 + }, + "loop_add_chain_4": { + "category": "loop", + "avg_instr_count": 31.0, + "runs": 3 + }, + "loop_relu_add_4": { + "category": "loop", + "avg_instr_count": 30.0, + "runs": 3 + }, + "dot_4": { + "category": "reduction", + "avg_instr_count": 8.0, + "runs": 3 + }, + "dot_8": { + "category": "reduction", + "avg_instr_count": 8.0, + "runs": 3 + }, + "dot_relu_4": { + "category": "reduction", + "avg_instr_count": 9.0, + "runs": 3 + }, + "dot_relu_8": { + "category": "reduction", + "avg_instr_count": 9.0, + "runs": 3 + }, + "matmul_2x2": { + "category": "tensor", + "avg_instr_count": 9.0, + "runs": 3 + }, + "matmul_4x4": { + "category": "tensor", + "avg_instr_count": 9.0, + "runs": 3 + }, + "matmul_add_2x2": { + "category": "tensor", + "avg_instr_count": 10.0, + "runs": 3 + }, + "matmul_relu_2x2": { + "category": "tensor", + "avg_instr_count": 10.0, + "runs": 3 + } +} \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/reports/course_report_instructions.png b/ScratchV-topic06-deliverable/reports/course_report_instructions.png new file mode 100644 index 0000000..d341442 Binary files /dev/null and b/ScratchV-topic06-deliverable/reports/course_report_instructions.png differ diff --git a/ScratchV-topic06-deliverable/reports/report.html b/ScratchV-topic06-deliverable/reports/report.html new file mode 100644 index 0000000..2ea5619 --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/report.html @@ -0,0 +1,286 @@ + + + + + ScratchV 课程版测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数:23,通过:23,失败:0,通过率:100.0%

+

测试目录:tests_main,性能退化阈值:5.0%

+
+ 课程版指令数图表 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
用例类别状态平均指令数95%置信区间变化率(%)是否退化描述
add_relu_reluactivationPASS9.00±0.000.00FalseAdd input and bias, then apply ReLU twice.
relu_addactivationPASS8.00±0.000.00FalseAdd input and bias, then apply one ReLU.
relu_onlyactivationPASS7.00±0.000.00FalseApply ReLU directly to a single input value.
relu_twiceactivationPASS8.00±0.000.00FalseApply ReLU twice to the same activation path.
if_elsebranchPASS26.00±0.000.00Falseif/else branch returns subtraction result when flag is zero.
if_relubranchPASS22.00±0.000.00Falseif/else branch combined with add and relu.
if_thenbranchPASS26.00±0.000.00Falseif/else branch returns add result when flag is non-zero.
add_chainelementwisePASS8.00±0.000.00FalseAdd a and b, then add c to the intermediate result.
add_chain_3elementwisePASS9.00±0.000.00FalseChain three add operations across four symbolic inputs.
add_fan_in_4elementwisePASS9.00±0.000.00FalseCompute two independent adds and then merge them with a final add.
add_reuseelementwisePASS8.00±0.000.00FalseReuse the same intermediate add result on both operands of a second add.
vector_addelementwisePASS7.00±0.000.00FalseSingle add over two symbolic vector inputs.
loop_add_4loopPASS27.00±0.000.00FalseRun a four-iteration loop whose body computes one add; final returned value is the last loop-body result.
loop_add_chain_4loopPASS31.00±0.000.00FalseRun a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.
loop_relu_add_4loopPASS30.00±0.000.00FalseRun a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.
dot_4reductionPASS8.00±0.000.00FalseCompute the dot product of two symbolic vectors of length 4.
dot_8reductionPASS8.00±0.000.00FalseCompute the dot product of two symbolic vectors of length 8.
dot_relu_4reductionPASS9.00±0.000.00FalseCompute a length-4 dot product and pass it through ReLU.
dot_relu_8reductionPASS9.00±0.000.00FalseCompute a length-8 dot product and pass it through ReLU.
matmul_2x2tensorPASS9.00±0.000.00FalseCompute a symbolic 2x2 by 2x2 matrix multiplication.
matmul_4x4tensorPASS9.00±0.000.00FalseCompute a symbolic 4x4 by 4x4 matrix multiplication.
matmul_add_2x2tensorPASS10.00±0.000.00FalseCompute a 2x2 matmul and then add a symbolic bias term.
matmul_relu_2x2tensorPASS10.00±0.000.00FalseCompute a 2x2 matmul and then apply ReLU to its result.
+ + \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/reports/report.md b/ScratchV-topic06-deliverable/reports/report.md new file mode 100644 index 0000000..c2112a8 --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/report.md @@ -0,0 +1,495 @@ +# ScratchV DSL 编译器性能测试报告 + +## 测试概览 + +- 生成时间: 2026-06-23 18:20:50 +- 用例总数: 23 +- 通过数量: 23 +- 失败数量: 0 +- 通过率: 100.0% +- 测试目录: `tests_main` +- 汇编输出目录: `build` +- 性能基线文件: `reports\benchmark_baseline.json` +- 性能退化阈值: 5.0% + +## 测试结果 + +| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 | +|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---| +| add_relu_relu | activation | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | 7 | 7 | True | build\add_relu_relu.s | +| relu_add | activation | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 3 | 3 | True | build\relu_add.s | +| relu_only | activation | PASS | stub | 7.00 | ±0.00 | 7 | 7 | 7.00 | 0.00 | False | 0 | 0 | True | build\relu_only.s | +| relu_twice | activation | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 4 | 4 | True | build\relu_twice.s | +| if_else | branch | PASS | stub | 26.00 | ±0.00 | 26 | 26 | 26.00 | 0.00 | False | 5 | 5 | True | build\if_else.s | +| if_relu | branch | PASS | stub | 22.00 | ±0.00 | 22 | 22 | 22.00 | 0.00 | False | 0 | 0 | True | build\if_relu.s | +| if_then | branch | PASS | stub | 26.00 | ±0.00 | 26 | 26 | 26.00 | 0.00 | False | 13 | 13 | True | build\if_then.s | +| add_chain | elementwise | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 9 | 9 | True | build\add_chain.s | +| add_chain_3 | elementwise | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | 14 | 14 | True | build\add_chain_3.s | +| add_fan_in_4 | elementwise | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | 10 | 10 | True | build\add_fan_in_4.s | +| add_reuse | elementwise | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 10 | 10 | True | build\add_reuse.s | +| vector_add | elementwise | PASS | stub | 7.00 | ±0.00 | 7 | 7 | 7.00 | 0.00 | False | 5 | 5 | True | build\vector_add.s | +| loop_add_4 | loop | PASS | stub | 27.00 | ±0.00 | 27 | 27 | 27.00 | 0.00 | False | 5 | 5 | True | build\loop_add_4.s | +| loop_add_chain_4 | loop | PASS | stub | 31.00 | ±0.00 | 31 | 31 | 31.00 | 0.00 | False | 9 | 9 | True | build\loop_add_chain_4.s | +| loop_relu_add_4 | loop | PASS | stub | 30.00 | ±0.00 | 30 | 30 | 30.00 | 0.00 | False | 2 | 2 | True | build\loop_relu_add_4.s | +| dot_4 | reduction | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 70 | 70 | True | build\dot_4.s | +| dot_8 | reduction | PASS | stub | 8.00 | ±0.00 | 8 | 8 | 8.00 | 0.00 | False | 36 | 36 | True | build\dot_8.s | +| dot_relu_4 | reduction | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | 0 | 0 | True | build\dot_relu_4.s | +| dot_relu_8 | reduction | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | 8 | 8 | True | build\dot_relu_8.s | +| matmul_2x2 | tensor | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | [[19, 22], [43, 50]] | [[19, 22], [43, 50]] | True | build\matmul_2x2.s | +| matmul_4x4 | tensor | PASS | stub | 9.00 | ±0.00 | 9 | 9 | 9.00 | 0.00 | False | [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] | [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] | True | build\matmul_4x4.s | +| matmul_add_2x2 | tensor | PASS | stub | 10.00 | ±0.00 | 10 | 10 | 10.00 | 0.00 | False | [[20, 23], [44, 51]] | [[20, 23], [44, 51]] | True | build\matmul_add_2x2.s | +| matmul_relu_2x2 | tensor | PASS | stub | 10.00 | ±0.00 | 10 | 10 | 10.00 | 0.00 | False | [[0, 2], [0, 4]] | [[0, 2], [0, 4]] | True | build\matmul_relu_2x2.s | + +## 性能图表 + +![课程版指令数图表](course_report_instructions.png) + +### Mermaid 图表 + +```mermaid +xychart-beta + title "各测试用例指令数" + x-axis ["add_relu_relu", "relu_add", "relu_only", "relu_twice", "if_else", "if_relu", "if_then", "add_chain", "add_chain_3", "add_fan_in_4", "add_reuse", "vector_add", "loop_add_4", "loop_add_chain_4", "loop_relu_add_4", "dot_4", "dot_8", "dot_relu_4", "dot_relu_8", "matmul_2x2", "matmul_4x4", "matmul_add_2x2", "matmul_relu_2x2"] + y-axis "指令数" 0 --> 33.0 + bar [9.0, 8.0, 7.0, 8.0, 26.0, 22.0, 26.0, 8.0, 9.0, 9.0, 8.0, 7.0, 27.0, 31.0, 30.0, 8.0, 8.0, 9.0, 9.0, 9.0, 9.0, 10.0, 10.0] +``` + +## 用例详情 + +### add_relu_relu + +- 类别: activation +- 描述: Add input and bias, then apply ReLU twice. +- 预期输出 (return_value): 7 +- 实际输出: 7 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\add_relu_relu.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### relu_add + +- 类别: activation +- 描述: Add input and bias, then apply one ReLU. +- 预期输出 (return_value): 3 +- 实际输出: 3 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\relu_add.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### relu_only + +- 类别: activation +- 描述: Apply ReLU directly to a single input value. +- 预期输出 (return_value): 0 +- 实际输出: 0 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\relu_only.s +- Benchmark 重复次数: 3 +- 平均指令数: 7.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 7 +- 最大指令数: 7 +- 基线指令数: 7.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### relu_twice + +- 类别: activation +- 描述: Apply ReLU twice to the same activation path. +- 预期输出 (return_value): 4 +- 实际输出: 4 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\relu_twice.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### if_else + +- 类别: branch +- 描述: if/else branch returns subtraction result when flag is zero. +- 预期输出 (return_value): 5 +- 实际输出: 5 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\if_else.s +- Benchmark 重复次数: 3 +- 平均指令数: 26.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 26 +- 最大指令数: 26 +- 基线指令数: 26.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### if_relu + +- 类别: branch +- 描述: if/else branch combined with add and relu. +- 预期输出 (return_value): 0 +- 实际输出: 0 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\if_relu.s +- Benchmark 重复次数: 3 +- 平均指令数: 22.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 22 +- 最大指令数: 22 +- 基线指令数: 22.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### if_then + +- 类别: branch +- 描述: if/else branch returns add result when flag is non-zero. +- 预期输出 (return_value): 13 +- 实际输出: 13 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\if_then.s +- Benchmark 重复次数: 3 +- 平均指令数: 26.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 26 +- 最大指令数: 26 +- 基线指令数: 26.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### add_chain + +- 类别: elementwise +- 描述: Add a and b, then add c to the intermediate result. +- 预期输出 (return_value): 9 +- 实际输出: 9 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\add_chain.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### add_chain_3 + +- 类别: elementwise +- 描述: Chain three add operations across four symbolic inputs. +- 预期输出 (return_value): 14 +- 实际输出: 14 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\add_chain_3.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### add_fan_in_4 + +- 类别: elementwise +- 描述: Compute two independent adds and then merge them with a final add. +- 预期输出 (return_value): 10 +- 实际输出: 10 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\add_fan_in_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### add_reuse + +- 类别: elementwise +- 描述: Reuse the same intermediate add result on both operands of a second add. +- 预期输出 (return_value): 10 +- 实际输出: 10 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\add_reuse.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### vector_add + +- 类别: elementwise +- 描述: Single add over two symbolic vector inputs. +- 预期输出 (return_value): 5 +- 实际输出: 5 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\vector_add.s +- Benchmark 重复次数: 3 +- 平均指令数: 7.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 7 +- 最大指令数: 7 +- 基线指令数: 7.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### loop_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result. +- 预期输出 (return_value): 5 +- 实际输出: 5 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\loop_add_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 27.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 27 +- 最大指令数: 27 +- 基线指令数: 27.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### loop_add_chain_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result. +- 预期输出 (return_value): 9 +- 实际输出: 9 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\loop_add_chain_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 31.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 31 +- 最大指令数: 31 +- 基线指令数: 31.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### loop_relu_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result. +- 预期输出 (return_value): 2 +- 实际输出: 2 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\loop_relu_add_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 30.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 30 +- 最大指令数: 30 +- 基线指令数: 30.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### dot_4 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 4. +- 预期输出 (return_value): 70 +- 实际输出: 70 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\dot_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### dot_8 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 8. +- 预期输出 (return_value): 36 +- 实际输出: 36 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\dot_8.s +- Benchmark 重复次数: 3 +- 平均指令数: 8.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 8 +- 最大指令数: 8 +- 基线指令数: 8.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### dot_relu_4 + +- 类别: reduction +- 描述: Compute a length-4 dot product and pass it through ReLU. +- 预期输出 (return_value): 0 +- 实际输出: 0 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\dot_relu_4.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### dot_relu_8 + +- 类别: reduction +- 描述: Compute a length-8 dot product and pass it through ReLU. +- 预期输出 (return_value): 8 +- 实际输出: 8 +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\dot_relu_8.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### matmul_2x2 + +- 类别: tensor +- 描述: Compute a symbolic 2x2 by 2x2 matrix multiplication. +- 预期输出 (return_value): [[19, 22], [43, 50]] +- 实际输出: [[19, 22], [43, 50]] +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\matmul_2x2.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### matmul_4x4 + +- 类别: tensor +- 描述: Compute a symbolic 4x4 by 4x4 matrix multiplication. +- 预期输出 (return_value): [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- 实际输出: [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\matmul_4x4.s +- Benchmark 重复次数: 3 +- 平均指令数: 9.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 9 +- 最大指令数: 9 +- 基线指令数: 9.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### matmul_add_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then add a symbolic bias term. +- 预期输出 (return_value): [[20, 23], [44, 51]] +- 实际输出: [[20, 23], [44, 51]] +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\matmul_add_2x2.s +- Benchmark 重复次数: 3 +- 平均指令数: 10.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 10 +- 最大指令数: 10 +- 基线指令数: 10.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + +### matmul_relu_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then apply ReLU to its result. +- 预期输出 (return_value): [[0, 2], [0, 4]] +- 实际输出: [[0, 2], [0, 4]] +- 输出是否匹配: True +- 模拟后端: stub +- 汇编文件: build\matmul_relu_2x2.s +- Benchmark 重复次数: 3 +- 平均指令数: 10.00 +- 95% 置信区间: ±0.00 +- 最小指令数: 10 +- 最大指令数: 10 +- 基线指令数: 10.00 +- 性能变化率: 0.00% +- 性能退化阈值: 5.00% +- 是否性能退化: False + diff --git a/ScratchV-topic06-deliverable/run_tests.py b/ScratchV-topic06-deliverable/run_tests.py new file mode 100644 index 0000000..e23d330 --- /dev/null +++ b/ScratchV-topic06-deliverable/run_tests.py @@ -0,0 +1,787 @@ +import argparse +import json +import math +import os +import subprocess +import sys +import tempfile +from datetime import datetime +from pathlib import Path + +from scratchv.simulator.tinyfive import verify_assembly + +TEST_DIR = Path("tests_main") +BUILD_DIR = Path("build") +REPORT_DIR = Path("reports") +REPORT_FILE = REPORT_DIR / "report.md" +HTML_REPORT_FILE = REPORT_DIR / "report.html" +CHART_FILE = REPORT_DIR / "course_report_instructions.png" +BASELINE_FILE = REPORT_DIR / "benchmark_baseline.json" +REGRESSION_THRESHOLD_PCT = 5.0 + + +def run_compile(dsl_file: Path): + output_file = BUILD_DIR / (dsl_file.stem + ".s") + + cmd = [ + sys.executable, + "-m", + "scratchv.main", + str(dsl_file), + "-o", + str(output_file), + "--optimize", + "all", + "--dump-ir", + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + ) + + return result, output_file + + +def load_metadata(dsl_file: Path): + meta_file = dsl_file.with_suffix(".meta.json") + if not meta_file.exists(): + return { + "description": "", + "expected_output_type": "return_value", + "expected_return": "", + } + return json.loads(meta_file.read_text(encoding="utf-8")) + + +def run_simulation(asm_file: Path): + if not asm_file.exists(): + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": "assembly file not found", + } + + asm_code = asm_file.read_text(encoding="utf-8") + return verify_assembly(asm_code) + + +def apply_add(lhs, rhs): + if isinstance(lhs, list) and isinstance(rhs, list): + return [apply_add(a, b) for a, b in zip(lhs, rhs)] + if isinstance(lhs, list): + return [apply_add(a, rhs) for a in lhs] + if isinstance(rhs, list): + return [apply_add(lhs, b) for b in rhs] + return lhs + rhs + + +def apply_binary(lhs, rhs, op): + if isinstance(lhs, list) and isinstance(rhs, list): + return [apply_binary(a, b, op) for a, b in zip(lhs, rhs)] + if isinstance(lhs, list): + return [apply_binary(a, rhs, op) for a in lhs] + if isinstance(rhs, list): + return [apply_binary(lhs, b, op) for b in rhs] + return op(lhs, rhs) + + +def apply_relu(value): + if isinstance(value, list): + return [apply_relu(v) for v in value] + return value if value > 0 else 0 + + +def apply_gelu(value): + if isinstance(value, list): + return [apply_gelu(v) for v in value] + return 0.5 * value * (1.0 + math.erf(value / math.sqrt(2.0))) + + +def apply_softmax(value): + if not isinstance(value, list): + return 1.0 + max_value = max(value) + exp_values = [math.exp(v - max_value) for v in value] + total = sum(exp_values) + return [v / total for v in exp_values] + + +def apply_maxpool(value, kernel, stride): + if not isinstance(value, list): + return value + return [max(value[i:i + kernel]) for i in range(0, len(value) - kernel + 1, stride)] + + +def apply_dot(lhs, rhs, length): + return sum(lhs[i] * rhs[i] for i in range(length)) + + +def apply_matmul(lhs, rhs, m, n, k): + result = [] + for i in range(m): + row = [] + for j in range(n): + cell = 0 + for kk in range(k): + cell += lhs[i][kk] * rhs[kk][j] + row.append(cell) + result.append(row) + return result + + +def resolve_value(token, env): + try: + return int(token) + except ValueError: + pass + try: + return float(token) + except ValueError: + pass + return env[token] + + +def execute_block(lines, env, start_idx=0, end_idx=None): + if end_idx is None: + end_idx = len(lines) + + idx = start_idx + while idx < end_idx: + line = lines[idx] + + if line.startswith("for "): + loop_var_text = line.replace("for ", "", 1) + loop_var, bounds = [p.strip() for p in loop_var_text.split("=", 1)] + loop_start_text, loop_end_text = [p.strip() for p in bounds.split(",", 1)] + loop_start = int(loop_start_text) + loop_end = int(loop_end_text) + + depth = 1 + body_start = idx + 1 + body_end = body_start + while body_end < end_idx and depth > 0: + current = lines[body_end] + if current.startswith("for "): + depth += 1 + elif current == "endfor": + depth -= 1 + if depth == 0: + break + body_end += 1 + + for i in range(loop_start, loop_end): + env[loop_var] = i + returned, value = execute_block(lines, env, body_start, body_end) + if returned: + return True, value + + idx = body_end + 1 + continue + + if line.startswith("if "): + cond_text = line.replace("if ", "", 1).strip() + cond_value = resolve_value(cond_text, env) + + depth = 1 + body_start = idx + 1 + scan_idx = body_start + else_idx = None + endif_idx = None + while scan_idx < end_idx: + current = lines[scan_idx] + if current.startswith("if "): + depth += 1 + elif current == "endif": + depth -= 1 + if depth == 0: + endif_idx = scan_idx + break + elif current == "else" and depth == 1: + else_idx = scan_idx + scan_idx += 1 + + if endif_idx is None: + raise ValueError("if without matching endif") + + if cond_value: + branch_start = body_start + branch_end = else_idx if else_idx is not None else endif_idx + else: + branch_start = else_idx + 1 if else_idx is not None else endif_idx + branch_end = endif_idx + + returned, value = execute_block(lines, env, branch_start, branch_end) + if returned: + return True, value + + idx = endif_idx + 1 + continue + + if line in {"else", "endif"}: + return False, None + + if line == "endfor": + return False, None + + if line.startswith("return "): + return True, resolve_value(line.replace("return ", "", 1).strip(), env) + + dest_name, expr = [p.strip() for p in line.split("=", 1)] + op_name = expr[:expr.index("(")] + arg_text = expr[expr.index("(") + 1: expr.rindex(")")] + args = [a.strip() for a in arg_text.split(",") if a.strip()] + + plain_args = [] + kwargs = {} + for arg in args: + if ":" in arg: + key, value = arg.split(":", 1) + kwargs[key.strip()] = int(value.strip()) + else: + plain_args.append(resolve_value(arg, env)) + + if op_name == "add": + env[dest_name] = apply_add(plain_args[0], plain_args[1]) + elif op_name == "sub": + env[dest_name] = apply_binary(plain_args[0], plain_args[1], lambda a, b: a - b) + elif op_name == "mul": + env[dest_name] = apply_binary(plain_args[0], plain_args[1], lambda a, b: a * b) + elif op_name == "div": + env[dest_name] = apply_binary(plain_args[0], plain_args[1], lambda a, b: a / b) + elif op_name == "relu": + env[dest_name] = apply_relu(plain_args[0]) + elif op_name == "gelu": + env[dest_name] = apply_gelu(plain_args[0]) + elif op_name == "softmax": + env[dest_name] = apply_softmax(plain_args[0]) + elif op_name == "maxpool": + env[dest_name] = apply_maxpool( + plain_args[0], + kwargs.get("kernel", 2), + kwargs.get("stride", 2), + ) + elif op_name == "dot": + env[dest_name] = apply_dot(plain_args[0], plain_args[1], kwargs["len"]) + elif op_name == "matmul": + env[dest_name] = apply_matmul( + plain_args[0], + plain_args[1], + kwargs["m"], + kwargs["n"], + kwargs["k"], + ) + else: + raise ValueError(f"Unsupported op in reference executor: {op_name}") + + idx += 1 + + return False, None + + +def execute_dsl_reference(dsl_file: Path, inputs): + raw_lines = dsl_file.read_text(encoding="utf-8").splitlines() + lines = [] + for raw_line in raw_lines: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + lines.append(line) + + env = dict(inputs) + returned, value = execute_block(lines, env) + return value if returned else None + + +def values_equal(lhs, rhs): + if isinstance(lhs, list) and isinstance(rhs, list): + if len(lhs) != len(rhs): + return False + return all(values_equal(a, b) for a, b in zip(lhs, rhs)) + if isinstance(lhs, float) or isinstance(rhs, float): + return math.isclose(lhs, rhs, rel_tol=1e-7, abs_tol=1e-7) + return lhs == rhs + + +def summarize_benchmark_runs(instr_counts): + if not instr_counts: + return { + "runs": 0, + "avg_instr_count": 0.0, + "min_instr_count": 0, + "max_instr_count": 0, + "ci95_instr_count": 0.0, + } + avg = sum(instr_counts) / len(instr_counts) + if len(instr_counts) > 1: + variance = sum((value - avg) ** 2 for value in instr_counts) / (len(instr_counts) - 1) + ci95 = 1.96 * math.sqrt(variance) / math.sqrt(len(instr_counts)) + else: + ci95 = 0.0 + return { + "runs": len(instr_counts), + "avg_instr_count": avg, + "min_instr_count": min(instr_counts), + "max_instr_count": max(instr_counts), + "ci95_instr_count": ci95, + } + + +def detect_regression(avg_instr_count, baseline_instr_count): + delta = avg_instr_count - baseline_instr_count + delta_pct = 0.0 if baseline_instr_count == 0 else (delta / baseline_instr_count) * 100.0 + return { + "baseline_instr_count": baseline_instr_count, + "delta": round(delta, 4), + "delta_pct": round(delta_pct, 4), + "threshold_pct": REGRESSION_THRESHOLD_PCT, + "regressed": delta_pct > REGRESSION_THRESHOLD_PCT, + } + + +def load_baseline(): + if not BASELINE_FILE.exists(): + return {} + return json.loads(BASELINE_FILE.read_text(encoding="utf-8")) + + +def save_baseline(results): + REPORT_DIR.mkdir(exist_ok=True) + payload = {} + for r in results: + payload[r["name"]] = { + "category": r["category"], + "avg_instr_count": r["avg_instr_count"], + "runs": r["benchmark_runs"], + } + BASELINE_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + +def generate_report_text(results, passed, failed): + lines = [] + lines.append("# ScratchV DSL 编译器性能测试报告\n\n") + + lines.append("## 测试概览\n\n") + lines.append(f"- 用例总数: {len(results)}\n") + lines.append(f"- 通过数量: {passed}\n") + lines.append(f"- 失败数量: {failed}\n\n") + + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + if benchmark_mode: + lines.append("## 性能基准概览\n\n") + lines.append("- 运行模式: benchmark\n") + lines.append(f"- 性能基线文件: `{BASELINE_FILE}`\n\n") + + lines.append("## 测试结果\n\n") + if benchmark_mode: + lines.append("| 测试用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 最小值 | 最大值 | 基线 | 变化率 | 是否退化 | 预期输出 | 实际输出 | 是否匹配 | 汇编文件 |\n") + lines.append("|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|\n") + else: + lines.append("| 测试用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 是否匹配 | 汇编文件 |\n") + lines.append("|---|---|---|---|---:|---|---|---|---|\n") + + for r in results: + expected = str(r["expected"]).replace("\n", " ").replace("|", "\\|") + actual = str(r["actual"]).replace("\n", " ").replace("|", "\\|") + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | {r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | {r['regressed']} | " + f"{expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.append("\n## 性能图表\n\n") + chart_cases = [r["name"] for r in results] + chart_instr = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.append("### 各测试用例指令数\n\n") + lines.append("```mermaid\n") + lines.append("xychart-beta\n") + lines.append(' title "各测试用例指令数"\n') + lines.append(" x-axis [" + ", ".join(f'"{name}"' for name in chart_cases) + "]\n") + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + lines.append(f' y-axis "指令数" 0 --> {max_instr + 2}\n') + lines.append(" bar [" + ", ".join(chart_instr) + "]\n") + lines.append("```\n\n") + + category_totals = {} + for r in results: + category_totals[r["category"]] = category_totals.get(r["category"], 0) + r.get("avg_instr_count", r["instr_count"]) + lines.append("### 各类别指令数占比\n\n") + lines.append("```mermaid\n") + lines.append("pie showData\n") + lines.append(' title 各类别指令数占比\n') + for category, total in sorted(category_totals.items()): + lines.append(f' "{category}" : {total}\n') + lines.append("```\n") + + lines.append("\n## 用例详情\n\n") + for r in results: + lines.append(f"### {r['name']}\n\n") + lines.append(f"- 类别: {r['category']}\n") + lines.append(f"- 描述: {r['description']}\n") + lines.append(f"- 预期输出 ({r['expected_type']}): {r['expected']}\n") + lines.append(f"- 实际输出: {r['actual']}\n") + lines.append(f"- 是否匹配: {r['matched']}\n") + lines.append(f"- 模拟后端: {r['backend']}\n") + if benchmark_mode: + lines.append(f"- Benchmark 重复次数: {r['benchmark_runs']}\n") + lines.append(f"- 平均指令数: {r['avg_instr_count']:.2f}\n") + lines.append(f"- 最小指令数: {r['min_instr_count']}\n") + lines.append(f"- 最大指令数: {r['max_instr_count']}\n") + lines.append(f"- 基线指令数: {r['baseline_instr_count']:.2f}\n") + lines.append(f"- 性能变化率 (%): {r['delta_pct']:.2f}\n") + lines.append(f"- 是否性能退化: {r['regressed']}\n") + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append(f"- 汇编文件: {r['asm']}\n\n") + + return "".join(lines) + + +def write_report(results, passed, failed): + REPORT_DIR.mkdir(exist_ok=True) + + REPORT_FILE.write_text(generate_report_text(results, passed, failed), encoding="utf-8") + print(f"\nReport written to {REPORT_FILE}") + + +def _markdown_cell(value): + return str(value).replace("\n", " ").replace("|", "\\|") + + +def generate_report_text(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | " + f"{r['regressed']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- 实际输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def write_chart(results): + try: + mpl_config_dir = Path(tempfile.gettempdir()) / "scratchv-matplotlib" + mpl_config_dir.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MPLCONFIGDIR", str(mpl_config_dir)) + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + return None + + names = [r["name"] for r in results] + values = [r.get("avg_instr_count", r["instr_count"]) for r in results] + width = max(10, len(names) * 0.45) + fig, ax = plt.subplots(figsize=(width, 5)) + ax.bar(range(len(names)), values, color="#2563eb") + ax.set_title("ScratchV Course Benchmark Instruction Counts") + ax.set_ylabel("Instructions") + ax.set_xticks(range(len(names))) + ax.set_xticklabels(names, rotation=60, ha="right", fontsize=8) + ax.grid(axis="y", linestyle="--", alpha=0.35) + fig.tight_layout() + fig.savefig(CHART_FILE, dpi=160) + plt.close(fig) + return CHART_FILE + + +def write_html_report(results, passed, failed): + try: + from jinja2 import Template + except ImportError: + return None + + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + template = Template(""" + + + + ScratchV 课程版测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数:{{ total }},通过:{{ passed }},失败:{{ failed }},通过率:{{ "%.1f"|format(pass_rate) }}%

+

测试目录:{{ test_dir }},性能退化阈值:{{ threshold }}%

+
+ 课程版指令数图表 + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} + +
用例类别状态平均指令数95%置信区间变化率(%)是否退化描述
{{ r.name }}{{ r.category }}{{ r.status }}{{ "%.2f"|format(r.avg_instr_count) }}±{{ "%.2f"|format(r.ci95_instr_count) }}{{ "%.2f"|format(r.delta_pct) }}{{ r.regressed }}{{ r.description }}
+ + +""") + HTML_REPORT_FILE.write_text( + template.render( + total=len(results), + passed=passed, + failed=failed, + pass_rate=pass_rate, + test_dir=str(TEST_DIR), + threshold=REGRESSION_THRESHOLD_PCT, + chart_name=CHART_FILE.name, + results=results, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def write_report(results, passed, failed): + REPORT_DIR.mkdir(exist_ok=True) + write_chart(results) + REPORT_FILE.write_text(generate_report_text(results, passed, failed), encoding="utf-8") + html_path = write_html_report(results, passed, failed) + print(f"\nMarkdown report written to {REPORT_FILE}") + if html_path: + print(f"HTML report written to {html_path}") + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="Run ScratchV DSL benchmark suite.") + parser.add_argument("--benchmark", type=int, default=0, metavar="N", + help="Run each case N times and report average instruction count.") + parser.add_argument("--update-baseline", action="store_true", + help="Write current benchmark averages to the baseline file.") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + BUILD_DIR.mkdir(exist_ok=True) + baseline = load_baseline() if args.benchmark else {} + + dsl_files = list(TEST_DIR.rglob("*.dsl")) + + if not dsl_files: + print("No DSL test files found.") + return + + passed = 0 + failed = 0 + results = [] + + print("Running DSL compiler tests...") + print("=" * 50) + + for dsl_file in dsl_files: + print(f"\n[TEST] {dsl_file}") + + meta = load_metadata(dsl_file) + result, output_file = run_compile(dsl_file) + sim_result = run_simulation(output_file) if result.returncode == 0 else { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": "compile failed", + } + expected_value = meta.get("expected_return") + actual_value = execute_dsl_reference(dsl_file, meta.get("inputs", {})) + matched = values_equal(actual_value, expected_value) + benchmark_counts = [] + benchmark_summary = { + "runs": 1, + "avg_instr_count": sim_result.get("instr_count", 0), + "min_instr_count": sim_result.get("instr_count", 0), + "max_instr_count": sim_result.get("instr_count", 0), + "ci95_instr_count": 0.0, + } + regression = { + "baseline_instr_count": 0.0, + "delta": 0.0, + "delta_pct": 0.0, + "threshold_pct": REGRESSION_THRESHOLD_PCT, + "regressed": False, + } + + if args.benchmark > 0 and result.returncode == 0 and output_file.exists(): + for _ in range(args.benchmark): + benchmark_counts.append(run_simulation(output_file).get("instr_count", 0)) + benchmark_summary = summarize_benchmark_runs(benchmark_counts) + baseline_entry = baseline.get(dsl_file.stem) + if baseline_entry: + regression = detect_regression( + avg_instr_count=benchmark_summary["avg_instr_count"], + baseline_instr_count=baseline_entry.get("avg_instr_count", 0.0), + ) + + ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and not regression["regressed"] + ) + + if ok: + print("PASS") + passed += 1 + status = "PASS" + else: + print("FAIL") + failed += 1 + status = "FAIL" + print(sim_result.get("error") or result.stderr or result.stdout) + + results.append({ + "name": dsl_file.stem, + "category": dsl_file.parent.name, + "path": str(dsl_file), + "status": status, + "description": meta.get("description", ""), + "expected_type": meta.get("expected_output_type", "return_value"), + "expected": expected_value, + "actual": actual_value, + "matched": matched, + "backend": sim_result.get("backend", "none"), + "instr_count": sim_result.get("instr_count", 0), + "benchmark_runs": benchmark_summary["runs"], + "avg_instr_count": benchmark_summary["avg_instr_count"], + "min_instr_count": benchmark_summary["min_instr_count"], + "max_instr_count": benchmark_summary["max_instr_count"], + "ci95_instr_count": benchmark_summary["ci95_instr_count"], + "baseline_instr_count": regression["baseline_instr_count"], + "delta": regression["delta"], + "delta_pct": regression["delta_pct"], + "threshold_pct": regression["threshold_pct"], + "regressed": regression["regressed"], + "asm": str(output_file), + }) + + print("\n" + "=" * 50) + print(f"Total: {len(dsl_files)}") + print(f"Passed: {passed}") + print(f"Failed: {failed}") + + if args.benchmark and args.update_baseline: + save_baseline(results) + print(f"Baseline written to {BASELINE_FILE}") + + write_report(results, passed, failed) + + +if __name__ == "__main__": + main() diff --git a/ScratchV-topic06-deliverable/setup.py b/ScratchV-topic06-deliverable/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/ScratchV-topic06-deliverable/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl new file mode 100644 index 0000000..52c1325 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl @@ -0,0 +1,5 @@ +# Add followed by two ReLU stages +x = add(input, bias) +y = relu(x) +result = relu(y) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json new file mode 100644 index 0000000..e587d8a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply ReLU twice.", + "expected_output_type": "return_value", + "inputs": { + "input": -3, + "bias": 10 + }, + "expected_return": 7 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl new file mode 100644 index 0000000..7e323b8 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl @@ -0,0 +1,4 @@ +# Add + ReLU activation +x = add(input, bias) +y = relu(x) +return y \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json new file mode 100644 index 0000000..f1b7f84 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply one ReLU.", + "expected_output_type": "return_value", + "inputs": { + "input": -2, + "bias": 5 + }, + "expected_return": 3 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl new file mode 100644 index 0000000..404a911 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl @@ -0,0 +1,3 @@ +# Single ReLU activation +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json new file mode 100644 index 0000000..b08af80 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU directly to a single input value.", + "expected_output_type": "return_value", + "inputs": { + "x": -5 + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl new file mode 100644 index 0000000..7d2defe --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl @@ -0,0 +1,4 @@ +# Two-stage ReLU activation +x = relu(input) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json new file mode 100644 index 0000000..fabbb38 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU twice to the same activation path.", + "expected_output_type": "return_value", + "inputs": { + "input": 4 + }, + "expected_return": 4 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl new file mode 100644 index 0000000..16f3c0d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl @@ -0,0 +1,8 @@ +# Branch takes the else path when flag is zero. +if flag +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json new file mode 100644 index 0000000..fdacf6a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 0, + "a": 9, + "b": 4 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl new file mode 100644 index 0000000..f748a83 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl @@ -0,0 +1,8 @@ +# Branch selects whether to apply ReLU after an add. +sum = add(a, b) +if use_relu +result = relu(sum) +return result +else +return sum +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json new file mode 100644 index 0000000..1925fb5 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch combined with add and relu.", + "expected_output_type": "return_value", + "inputs": { + "use_relu": 1, + "a": -8, + "b": 3 + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl new file mode 100644 index 0000000..b4f72d4 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl @@ -0,0 +1,8 @@ +# Branch takes the then path when flag is non-zero. +if flag +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json new file mode 100644 index 0000000..04748d1 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns add result when flag is non-zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 1, + "a": 9, + "b": 4 + }, + "expected_return": 13 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl new file mode 100644 index 0000000..e39db66 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl @@ -0,0 +1,4 @@ +# Chain of two vector adds +x = add(a, b) +y = add(x, c) +return y \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json new file mode 100644 index 0000000..b07adaf --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Add a and b, then add c to the intermediate result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl new file mode 100644 index 0000000..aeca4df --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl @@ -0,0 +1,5 @@ +# Chain of three add operations +x = add(a, b) +y = add(x, c) +z = add(y, d) +return z diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json new file mode 100644 index 0000000..368b3aa --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Chain three add operations across four symbolic inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4, + "d": 5 + }, + "expected_return": 14 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl new file mode 100644 index 0000000..0e4449e --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl @@ -0,0 +1,5 @@ +# Fan-in add over four inputs +x = add(a, b) +y = add(c, d) +result = add(x, y) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json new file mode 100644 index 0000000..3eb1f4a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute two independent adds and then merge them with a final add.", + "expected_output_type": "return_value", + "inputs": { + "a": 1, + "b": 2, + "c": 3, + "d": 4 + }, + "expected_return": 10 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl new file mode 100644 index 0000000..4618948 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl @@ -0,0 +1,4 @@ +# Reuse intermediate add result +x = add(a, b) +result = add(x, x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json new file mode 100644 index 0000000..2d2c396 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 10 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl new file mode 100644 index 0000000..129f923 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl @@ -0,0 +1,3 @@ +# Vector add +result = add(a, b) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json new file mode 100644 index 0000000..7347fff --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Single add over two symbolic vector inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl new file mode 100644 index 0000000..381f1c0 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl @@ -0,0 +1,5 @@ +# Loop with repeated add body over 4 iterations +for i = 0, 4 +x = add(a, b) +endfor +return x diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json new file mode 100644 index 0000000..fc31fae --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl new file mode 100644 index 0000000..eca595b --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl @@ -0,0 +1,6 @@ +# Loop with chained add inside the body +for i = 0, 4 +x = add(a, b) +y = add(x, c) +endfor +return y diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json new file mode 100644 index 0000000..154d91d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl new file mode 100644 index 0000000..44f3080 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl @@ -0,0 +1,6 @@ +# Loop with add followed by ReLU in the body +for i = 0, 4 +x = add(input, bias) +y = relu(x) +endfor +return y diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json new file mode 100644 index 0000000..8baca28 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "input": -4, + "bias": 6 + }, + "expected_return": 2 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl new file mode 100644 index 0000000..7c02794 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl @@ -0,0 +1,3 @@ +# Dot product of two 4-element vectors +result = dot(a, b, len:4) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json new file mode 100644 index 0000000..748f260 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4], + "b": [5, 6, 7, 8] + }, + "expected_return": 70 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl new file mode 100644 index 0000000..e1b71fa --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl @@ -0,0 +1,3 @@ +# Dot product of two 8-element vectors +result = dot(a, b, len:8) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json new file mode 100644 index 0000000..d3172a2 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4, 5, 6, 7, 8], + "b": [1, 1, 1, 1, 1, 1, 1, 1] + }, + "expected_return": 36 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl new file mode 100644 index 0000000..279fc19 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl @@ -0,0 +1,4 @@ +# Dot product followed by ReLU +x = dot(a, b, len:4) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json new file mode 100644 index 0000000..24910bc --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, -2, 3, -4], + "b": [2, 3, 4, 5] + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl new file mode 100644 index 0000000..d2b906d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl @@ -0,0 +1,4 @@ +# Dot product of length 8 followed by ReLU +x = dot(a, b, len:8) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json new file mode 100644 index 0000000..776fab5 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 0, 1, 0, 1, 0, 1, 0], + "b": [2, 2, 2, 2, 2, 2, 2, 2] + }, + "expected_return": 8 +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl new file mode 100644 index 0000000..30307f1 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 2x2 * 2x2 +result = matmul(A, B, m:2, n:2, k:2) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json new file mode 100644 index 0000000..f970a61 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_output_type": "return_value", + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl new file mode 100644 index 0000000..ab5a865 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 4x4 * 4x4 +result = matmul(A, B, m:4, n:4, k:4) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json new file mode 100644 index 0000000..ead52b9 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_output_type": "return_value", + "inputs": { + "A": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], + "B": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] + }, + "expected_return": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl new file mode 100644 index 0000000..d41a2cb --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by add +x = matmul(A, B, m:2, n:2, k:2) +result = add(x, bias) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json new file mode 100644 index 0000000..9bca125 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_output_type": "return_value", + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]], + "bias": [[1, 1], [1, 1]] + }, + "expected_return": [[20, 23], [44, 51]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl new file mode 100644 index 0000000..28c7c61 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by ReLU +x = matmul(A, B, m:2, n:2, k:2) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json new file mode 100644 index 0000000..352dfa6 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_output_type": "return_value", + "inputs": { + "A": [[-1, 2], [-3, 4]], + "B": [[1, 0], [0, 1]] + }, + "expected_return": [[0, 2], [0, 4]] +}