diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd0c4f8..b0fbd51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,15 +5,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: 检出 kvlang(提供 kvlang / kvlanglayout 工具链源) - run: git clone --depth 1 https://github.com/array2d/kvlang.git ../kvlang - - name: 下载 ABI 依赖(kvspace / kvspace-durable .so → /usr/lib) + - name: 安装 kvlang(唯一依赖:自包含 release → /usr) env: { GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" } run: ./ci/deps.sh - - name: 构建并安装 kvlang 工具链(kvlang / kvlanglayout + .so → /usr) - working-directory: ../kvlang - run: | - make runtime runtime-rs layout - sudo -E make install - name: byteseek 自检(无网络、无 LLM,shm 后端) run: KVSPACE=shm:///tmp/byteseek_ci make test diff --git a/Makefile b/Makefile index eb50c76..29715eb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # byteseek 全 kvlang(无 Rust,无编译产物)。byteseek 不是可执行文件,而是活在 kvspace 里的 # 一棵 .kv 代码树,由标准 kvlang 工具链(kvlang / kvlanglayout)驱动。 -# deps 下载 ABI .so 到 /usr/lib(kvlang 二进制需另装,见 README) +# deps 装 kvlang 最新 release(bin/kvlang·kvlanglayout·kvspace + 库 + 头 → /usr) # boot layout lib/ 全部 .kv 进 kvspace 并执行各 init(config/语法速览/系统提示种入) # run boot 后进入 REPL(kvlang byteseek·main) # test 无网络无 LLM 自检(boot → layout tests/selftest.kv → run selftest·go) diff --git a/agentbench/00/arithmetic/add-mult.answer b/agentbench/00/arithmetic/add-mult.answer deleted file mode 100644 index 4099407..0000000 --- a/agentbench/00/arithmetic/add-mult.answer +++ /dev/null @@ -1 +0,0 @@ -23 diff --git a/agentbench/00/arithmetic/add-mult.question b/agentbench/00/arithmetic/add-mult.question deleted file mode 100644 index b45851f..0000000 --- a/agentbench/00/arithmetic/add-mult.question +++ /dev/null @@ -1 +0,0 @@ -计算 3 + 4 × 5,把结果打印出来。 diff --git a/agentbench/00/arithmetic/int-div.answer b/agentbench/00/arithmetic/int-div.answer deleted file mode 100644 index 8351c19..0000000 --- a/agentbench/00/arithmetic/int-div.answer +++ /dev/null @@ -1 +0,0 @@ -14 diff --git a/agentbench/00/arithmetic/int-div.question b/agentbench/00/arithmetic/int-div.question deleted file mode 100644 index 9492762..0000000 --- a/agentbench/00/arithmetic/int-div.question +++ /dev/null @@ -1 +0,0 @@ -计算 100 ÷ 7 的整数商(整除结果),并打印。 diff --git a/agentbench/00/string/concat.answer b/agentbench/00/string/concat.answer deleted file mode 100644 index 3b18e51..0000000 --- a/agentbench/00/string/concat.answer +++ /dev/null @@ -1 +0,0 @@ -hello world diff --git a/agentbench/00/string/concat.question b/agentbench/00/string/concat.question deleted file mode 100644 index 67f7527..0000000 --- a/agentbench/00/string/concat.question +++ /dev/null @@ -1 +0,0 @@ -把字符串 "hello" 和 " world" 拼接成一个字符串并打印。 diff --git a/agentbench/01/arithmetic/sum-1-100.answer b/agentbench/01/arithmetic/sum-1-100.answer deleted file mode 100644 index c3ac783..0000000 --- a/agentbench/01/arithmetic/sum-1-100.answer +++ /dev/null @@ -1 +0,0 @@ -5050 diff --git a/agentbench/01/arithmetic/sum-1-100.question b/agentbench/01/arithmetic/sum-1-100.question deleted file mode 100644 index c8d49f3..0000000 --- a/agentbench/01/arithmetic/sum-1-100.question +++ /dev/null @@ -1 +0,0 @@ -求 1 到 100 之间所有整数的和(含 1 和 100),打印结果。 diff --git a/agentbench/01/string/reverse.answer b/agentbench/01/string/reverse.answer deleted file mode 100644 index 59d2d1c..0000000 --- a/agentbench/01/string/reverse.answer +++ /dev/null @@ -1 +0,0 @@ -cba diff --git a/agentbench/01/string/reverse.question b/agentbench/01/string/reverse.question deleted file mode 100644 index f428dc9..0000000 --- a/agentbench/01/string/reverse.question +++ /dev/null @@ -1 +0,0 @@ -把字符串 abc 反转后打印。 diff --git a/agentbench/01/tool-shell/echo.answer b/agentbench/01/tool-shell/echo.answer deleted file mode 100644 index 6fc6dca..0000000 --- a/agentbench/01/tool-shell/echo.answer +++ /dev/null @@ -1 +0,0 @@ -byteseek diff --git a/agentbench/01/tool-shell/echo.question b/agentbench/01/tool-shell/echo.question deleted file mode 100644 index bfed943..0000000 --- a/agentbench/01/tool-shell/echo.question +++ /dev/null @@ -1 +0,0 @@ -用 shell 打印字符串 byteseek。 diff --git a/agentbench/02/array/max.answer b/agentbench/02/array/max.answer deleted file mode 100644 index ec63514..0000000 --- a/agentbench/02/array/max.answer +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/agentbench/02/array/max.question b/agentbench/02/array/max.question deleted file mode 100644 index ef06cdc..0000000 --- a/agentbench/02/array/max.question +++ /dev/null @@ -1 +0,0 @@ -给定数组 [7, 2, 9, 4],找出其中的最大值并打印。 diff --git a/agentbench/02/tool-python/pow.answer b/agentbench/02/tool-python/pow.answer deleted file mode 100644 index d7b1c44..0000000 --- a/agentbench/02/tool-python/pow.answer +++ /dev/null @@ -1 +0,0 @@ -1024 diff --git a/agentbench/02/tool-python/pow.question b/agentbench/02/tool-python/pow.question deleted file mode 100644 index 19a82ca..0000000 --- a/agentbench/02/tool-python/pow.question +++ /dev/null @@ -1 +0,0 @@ -用 python 计算 2 的 10 次方并打印结果。 diff --git a/agentbench/02/tool-shell/kernel.answer b/agentbench/02/tool-shell/kernel.answer deleted file mode 100644 index 3f255c9..0000000 --- a/agentbench/02/tool-shell/kernel.answer +++ /dev/null @@ -1 +0,0 @@ -期望(冒烟): 输出等于 `uname -s`(本平台为 Linux) diff --git a/agentbench/02/tool-shell/kernel.question b/agentbench/02/tool-shell/kernel.question deleted file mode 100644 index e009a9d..0000000 --- a/agentbench/02/tool-shell/kernel.question +++ /dev/null @@ -1 +0,0 @@ -用 shell 打印当前操作系统的内核名称(uname)。 diff --git a/agentbench/03/array/sum.answer b/agentbench/03/array/sum.answer deleted file mode 100644 index 60d3b2f..0000000 --- a/agentbench/03/array/sum.answer +++ /dev/null @@ -1 +0,0 @@ -15 diff --git a/agentbench/03/array/sum.question b/agentbench/03/array/sum.question deleted file mode 100644 index 86d925a..0000000 --- a/agentbench/03/array/sum.question +++ /dev/null @@ -1 +0,0 @@ -给定数组 [1, 2, 3, 4, 5],求所有元素之和并打印。 diff --git a/agentbench/03/string/char-count.answer b/agentbench/03/string/char-count.answer deleted file mode 100644 index 00750ed..0000000 --- a/agentbench/03/string/char-count.answer +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/agentbench/03/string/char-count.question b/agentbench/03/string/char-count.question deleted file mode 100644 index da5a401..0000000 --- a/agentbench/03/string/char-count.question +++ /dev/null @@ -1 +0,0 @@ -统计字符串 banana 中字符 a 出现的次数并打印。 diff --git a/agentbench/03/tool-http/fetch-example.answer b/agentbench/03/tool-http/fetch-example.answer deleted file mode 100644 index 82d7471..0000000 --- a/agentbench/03/tool-http/fetch-example.answer +++ /dev/null @@ -1 +0,0 @@ -期望(冒烟): 输出 yes(响应正文含 "Example Domain") diff --git a/agentbench/03/tool-http/fetch-example.question b/agentbench/03/tool-http/fetch-example.question deleted file mode 100644 index 4c68318..0000000 --- a/agentbench/03/tool-http/fetch-example.question +++ /dev/null @@ -1 +0,0 @@ -用 http 抓取 https://example.com 的正文,判断其中是否包含 "Example Domain",打印 yes 或 no。 diff --git a/agentbench/04/array/second-max.answer b/agentbench/04/array/second-max.answer deleted file mode 100644 index 1e8b314..0000000 --- a/agentbench/04/array/second-max.answer +++ /dev/null @@ -1 +0,0 @@ -6 diff --git a/agentbench/04/array/second-max.question b/agentbench/04/array/second-max.question deleted file mode 100644 index 82382ce..0000000 --- a/agentbench/04/array/second-max.question +++ /dev/null @@ -1 +0,0 @@ -给定数组 [3, 1, 4, 1, 5, 9, 2, 6],找出第二大(严格小于最大值的最大元素)并打印。 diff --git a/agentbench/04/dict/read-member.answer b/agentbench/04/dict/read-member.answer deleted file mode 100644 index 671a444..0000000 --- a/agentbench/04/dict/read-member.answer +++ /dev/null @@ -1 +0,0 @@ -kv diff --git a/agentbench/04/dict/read-member.question b/agentbench/04/dict/read-member.question deleted file mode 100644 index f683da1..0000000 --- a/agentbench/04/dict/read-member.question +++ /dev/null @@ -1 +0,0 @@ -创建一个成员为 name="kv"、ver=1 的结构,打印其 name 成员的值。 diff --git a/agentbench/04/string/palindrome.answer b/agentbench/04/string/palindrome.answer deleted file mode 100644 index 7cfab5b..0000000 --- a/agentbench/04/string/palindrome.answer +++ /dev/null @@ -1 +0,0 @@ -yes diff --git a/agentbench/04/string/palindrome.question b/agentbench/04/string/palindrome.question deleted file mode 100644 index 76b3701..0000000 --- a/agentbench/04/string/palindrome.question +++ /dev/null @@ -1 +0,0 @@ -判断字符串 racecar 是否为回文(正反读相同),是打印 yes,否打印 no。 diff --git a/agentbench/05/dict/linked-list.answer b/agentbench/05/dict/linked-list.answer deleted file mode 100644 index 00750ed..0000000 --- a/agentbench/05/dict/linked-list.answer +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/agentbench/05/dict/linked-list.question b/agentbench/05/dict/linked-list.question deleted file mode 100644 index 7ec3ca6..0000000 --- a/agentbench/05/dict/linked-list.question +++ /dev/null @@ -1 +0,0 @@ -用路径指针构造两节点链表:/n1 的 val=1、next="/n2",/n2 的 val=2、next=""。从 /n1 遍历到末尾,打印沿途所有 val 之和。 diff --git a/agentbench/05/kvspace/persist.answer b/agentbench/05/kvspace/persist.answer deleted file mode 100644 index d81cc07..0000000 --- a/agentbench/05/kvspace/persist.answer +++ /dev/null @@ -1 +0,0 @@ -42 diff --git a/agentbench/05/kvspace/persist.question b/agentbench/05/kvspace/persist.question deleted file mode 100644 index 0756b19..0000000 --- a/agentbench/05/kvspace/persist.question +++ /dev/null @@ -1 +0,0 @@ -把值 42 写入绝对路径 /bench/answer,再从同一路径读回并打印。 diff --git a/agentbench/05/reasoning/cats.answer b/agentbench/05/reasoning/cats.answer deleted file mode 100644 index 00750ed..0000000 --- a/agentbench/05/reasoning/cats.answer +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/agentbench/05/reasoning/cats.question b/agentbench/05/reasoning/cats.question deleted file mode 100644 index 30c0d5e..0000000 --- a/agentbench/05/reasoning/cats.question +++ /dev/null @@ -1 +0,0 @@ -四只猫 A/B/C/D 体重分别为 3/5/7/9 公斤(各不同)。已知:A 不是最轻的,B 比 C 重,D 是 9 公斤。问 C 多重(只打印数字)。 diff --git a/agentbench/06/array/binary-search.answer b/agentbench/06/array/binary-search.answer deleted file mode 100644 index 00750ed..0000000 --- a/agentbench/06/array/binary-search.answer +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/agentbench/06/array/binary-search.question b/agentbench/06/array/binary-search.question deleted file mode 100644 index da43b5a..0000000 --- a/agentbench/06/array/binary-search.question +++ /dev/null @@ -1 +0,0 @@ -在有序数组 [1, 3, 5, 7, 9, 11, 13] 中查找 7 的下标(0 起),打印该下标。 diff --git a/agentbench/06/kvspace/multi-write.answer b/agentbench/06/kvspace/multi-write.answer deleted file mode 100644 index 1e8b314..0000000 --- a/agentbench/06/kvspace/multi-write.answer +++ /dev/null @@ -1 +0,0 @@ -6 diff --git a/agentbench/06/kvspace/multi-write.question b/agentbench/06/kvspace/multi-write.question deleted file mode 100644 index fde13c7..0000000 --- a/agentbench/06/kvspace/multi-write.question +++ /dev/null @@ -1 +0,0 @@ -把三个键值对 a=1、b=2、c=3 写入 /bench/ 下,然后读回三个值,打印它们的和。 diff --git a/agentbench/06/planning/pipe-multitool.answer b/agentbench/06/planning/pipe-multitool.answer deleted file mode 100644 index 59d2d1c..0000000 --- a/agentbench/06/planning/pipe-multitool.answer +++ /dev/null @@ -1 +0,0 @@ -cba diff --git a/agentbench/06/planning/pipe-multitool.question b/agentbench/06/planning/pipe-multitool.question deleted file mode 100644 index 5be2efe..0000000 --- a/agentbench/06/planning/pipe-multitool.question +++ /dev/null @@ -1 +0,0 @@ -先用 shell 打印字符串 abc,再用 python 把它反转,打印最终结果。要求经过 shell 与 python 两个工具。 diff --git a/agentbench/07/kvspace/nested-tree.answer b/agentbench/07/kvspace/nested-tree.answer deleted file mode 100644 index 190a180..0000000 --- a/agentbench/07/kvspace/nested-tree.answer +++ /dev/null @@ -1 +0,0 @@ -123 diff --git a/agentbench/07/kvspace/nested-tree.question b/agentbench/07/kvspace/nested-tree.question deleted file mode 100644 index 4b19d15..0000000 --- a/agentbench/07/kvspace/nested-tree.question +++ /dev/null @@ -1 +0,0 @@ -在 /bench/ 下写入嵌套结构 /bench/a/b/c=123,然后读回最深层 c 的值并打印。 diff --git a/agentbench/07/planning/fetch-parse.answer b/agentbench/07/planning/fetch-parse.answer deleted file mode 100644 index ae90161..0000000 --- a/agentbench/07/planning/fetch-parse.answer +++ /dev/null @@ -1 +0,0 @@ -期望(冒烟): 输出 Sample Slide Show diff --git a/agentbench/07/planning/fetch-parse.question b/agentbench/07/planning/fetch-parse.question deleted file mode 100644 index 023c7cc..0000000 --- a/agentbench/07/planning/fetch-parse.question +++ /dev/null @@ -1 +0,0 @@ -用 http 抓取 https://httpbin.org/json 的正文,用 json 解析出顶层 slideshow.title 字段并打印。 diff --git a/agentbench/07/reasoning/cities.answer b/agentbench/07/reasoning/cities.answer deleted file mode 100644 index 8bf3bb7..0000000 --- a/agentbench/07/reasoning/cities.answer +++ /dev/null @@ -1 +0,0 @@ -深圳 diff --git a/agentbench/07/reasoning/cities.question b/agentbench/07/reasoning/cities.question deleted file mode 100644 index 9f99a60..0000000 --- a/agentbench/07/reasoning/cities.question +++ /dev/null @@ -1 +0,0 @@ -五个学生分别来自北京/上海/广州/深圳/杭州(各不同)。已知:A 不是北京也不是深圳,B 是广州,C 不是深圳也不是杭州,D 是北京。问 E 来自哪里(只打印城市名)。 diff --git a/agentbench/08/meta/self-gen-llm.answer b/agentbench/08/meta/self-gen-llm.answer deleted file mode 100644 index afe4629..0000000 --- a/agentbench/08/meta/self-gen-llm.answer +++ /dev/null @@ -1 +0,0 @@ -期望(冒烟): 输出 42(需 DEEPSEEK_API_KEY) diff --git a/agentbench/08/meta/self-gen-llm.question b/agentbench/08/meta/self-gen-llm.question deleted file mode 100644 index af286c5..0000000 --- a/agentbench/08/meta/self-gen-llm.question +++ /dev/null @@ -1 +0,0 @@ -调用 llm.call 生成一段计算 6 × 7 并打印的程序,再执行它,打印最终结果。 diff --git a/agentbench/08/meta/self-inspect.answer b/agentbench/08/meta/self-inspect.answer deleted file mode 100644 index a70062b..0000000 --- a/agentbench/08/meta/self-inspect.answer +++ /dev/null @@ -1 +0,0 @@ -期望(冒烟): 输出 yes(需完整 bootstrap,kvlangbrief 已播种) diff --git a/agentbench/08/meta/self-inspect.question b/agentbench/08/meta/self-inspect.question deleted file mode 100644 index 991adfc..0000000 --- a/agentbench/08/meta/self-inspect.question +++ /dev/null @@ -1 +0,0 @@ -读回 /lib/byteseek.kvlangbrief 的内容,判断其长度是否大于 100,打印 yes 或 no。 diff --git a/agentbench/08/planning/error-recovery.answer b/agentbench/08/planning/error-recovery.answer deleted file mode 100644 index df881dd..0000000 --- a/agentbench/08/planning/error-recovery.answer +++ /dev/null @@ -1 +0,0 @@ -recovered diff --git a/agentbench/08/planning/error-recovery.question b/agentbench/08/planning/error-recovery.question deleted file mode 100644 index 3c35bd7..0000000 --- a/agentbench/08/planning/error-recovery.question +++ /dev/null @@ -1 +0,0 @@ -尝试用 shell 执行一个必然失败的命令 nonexistent_cmd_xyz,不因失败中断,最终打印 recovered。 diff --git a/agentbench/09/meta/memory-cross.answer b/agentbench/09/meta/memory-cross.answer deleted file mode 100644 index 3042f8d..0000000 --- a/agentbench/09/meta/memory-cross.answer +++ /dev/null @@ -1 +0,0 @@ -最喜欢的颜色是蓝色 diff --git a/agentbench/09/meta/memory-cross.question b/agentbench/09/meta/memory-cross.question deleted file mode 100644 index fc27465..0000000 --- a/agentbench/09/meta/memory-cross.question +++ /dev/null @@ -1 +0,0 @@ -把一段记忆「最喜欢的颜色是蓝色」写入 /mem/fav(模拟持久化记忆),再从 /mem/fav 读回并打印。 diff --git a/agentbench/09/planning/primes.answer b/agentbench/09/planning/primes.answer deleted file mode 100644 index 3f204e1..0000000 --- a/agentbench/09/planning/primes.answer +++ /dev/null @@ -1 +0,0 @@ -2 3 5 7 11 13 17 19 23 29 diff --git a/agentbench/09/planning/primes.question b/agentbench/09/planning/primes.question deleted file mode 100644 index 81a1ff3..0000000 --- a/agentbench/09/planning/primes.question +++ /dev/null @@ -1 +0,0 @@ -不借助网络,用 shell/python 组合求出前 10 个质数,按从小到大打印(空格分隔)。 diff --git a/agentbench/09/reasoning/mislabeled-boxes.answer b/agentbench/09/reasoning/mislabeled-boxes.answer deleted file mode 100644 index 1ad95a3..0000000 --- a/agentbench/09/reasoning/mislabeled-boxes.answer +++ /dev/null @@ -1 +0,0 @@ -混装 diff --git a/agentbench/09/reasoning/mislabeled-boxes.question b/agentbench/09/reasoning/mislabeled-boxes.question deleted file mode 100644 index 1e729b8..0000000 --- a/agentbench/09/reasoning/mislabeled-boxes.question +++ /dev/null @@ -1 +0,0 @@ -三个盒子分别装苹果、橘子、混装(苹果+橘子),但三个标签全贴错了。你只能从一个盒子里摸出一个水果(不能看里面)来判断三个盒子的真实内容。问应该摸贴哪个标签的盒子(打印:苹果 / 橘子 / 混装)。 diff --git a/agentbench/10/meta/open-goal.answer b/agentbench/10/meta/open-goal.answer deleted file mode 100644 index 4c339d8..0000000 --- a/agentbench/10/meta/open-goal.answer +++ /dev/null @@ -1 +0,0 @@ -评分标准: 能力定义清晰;有可运行 kvlang 代码;输出可复现验证;改动落在 kvspace 树内可观测。 diff --git a/agentbench/10/meta/open-goal.question b/agentbench/10/meta/open-goal.question deleted file mode 100644 index 3f47549..0000000 --- a/agentbench/10/meta/open-goal.question +++ /dev/null @@ -1 +0,0 @@ -定义一项你认为属于「智能」的新能力(如新工具、新记忆策略、新自改进协议),在 kvspace 中实现并演示它:说明思路、写出 kvlang 代码、给出可验证的输出。 diff --git a/agentbench/10/planning/full-autonomy.answer b/agentbench/10/planning/full-autonomy.answer deleted file mode 100644 index 6c61782..0000000 --- a/agentbench/10/planning/full-autonomy.answer +++ /dev/null @@ -1 +0,0 @@ -评分标准: 读入→排序→持久化→读回全链路正确;/bench/sorted 最终状态与打印一致;步骤说明清晰。 diff --git a/agentbench/10/planning/full-autonomy.question b/agentbench/10/planning/full-autonomy.question deleted file mode 100644 index 52562cf..0000000 --- a/agentbench/10/planning/full-autonomy.question +++ /dev/null @@ -1 +0,0 @@ -从零实现一个完整小任务:读入三个数、升序排序、把结果写入 /bench/sorted、再读回打印,全程用 kvspace 持久化,并说明每一步的作用。 diff --git a/agentbench/README.md b/agentbench/README.md index 90ab6a9..95eeea2 100644 --- a/agentbench/README.md +++ b/agentbench/README.md @@ -1,58 +1,105 @@ -# agentbench —— byteseek agent 能力测试基准 +# agentbench —— HumanEval / SWE-bench 基准入口 -对 corebrain(代码脑)做端到端能力测试:给定自然语言任务,agent 生成 kvlang 程序执行, -结果经 `println` 落到终端。每题两个文件:`.question`(任务)与 `.answer`(基准答案)。 - -## 目录结构 +agentbench 不再自带题目。它是一个**薄入口**:调用业界两套标准基准(Codex / Claude 评测用的 +HumanEval 系列与 SWE-bench 系列),把 byteseek 当作被测 agent 接进去。 ``` agentbench/ - {难度00..10}/{类别}/{name}.question - {name}.answer + bench 入口脚本(bash 壳,exec 到 BENCH_HOME 的 venv) + bench.py 驱动:题库 → agent → predictions → 官方评测 ``` -难度在前、类别在后,两级子目录。难度用两位补零 `00`–`10`(共 11 级)。 +## 快速开始 + +```bash +agentbench/bench env # 环境自检 +agentbench/bench list # suite 与题量 +agentbench/bench all humaneval --agent gold # 验管道(题库自带答案,不调 LLM) +agentbench/bench run humaneval --agent byteseek --limit 10 +agentbench/bench eval humaneval +``` -## 难度量表(0–10) +## 命令 -| 难度 | 说明 | +| 命令 | 作用 | |------|------| -| 0 | 单步、无工具:直接算术 / 字符串拼接 | -| 1 | 单步单工具(shell/python),或简单循环累加 | -| 2 | 单工具带参数 / 单次算法(数组最值、幂运算) | -| 3 | 单算法 / 字符串统计 / 网络冒烟 | -| 4 | 数组算法 / 回文判断 / dict 成员读 | -| 5 | dict 指针链表 / kvspace 读写 / 中等逻辑 | -| 6 | 多工具组合 / kvspace 多键 / 二分查找 | -| 7 | 抓取→解析→存储 / 嵌套状态树 / 多约束推理 | -| 8 | 自省自改 / 错误恢复 / 嵌套代码生成(需 LLM) | -| 9 | 跨会话记忆 / 开放分解 / 经典谜题 | -| 10 | 开放目标,无固定解(按 rubric 评分) | - -## 类别 - -| 类别 | 内容 | +| `bench env` | 打印框架版本、题库路径与题量、docker、kvlang/kvspace/byteseek 就绪状态 | +| `bench list` | 列出 suite 与题量 | +| `bench run ` | 生成 `predictions.jsonl`(agent 阶段) | +| `bench eval ` | 官方评测(eval 阶段);默认取该 suite 最近一次 run 的 predictions | +| `bench all ` | run + eval | + +`run` 选项:`--agent gold|byteseek`、`--limit N`、`--offset N`、`--task id1,id2`、 +`--out DIR`、`--timeout S`(单题 agent 超时)、`--no-boot`。 +`eval` 选项:`--predictions FILE`、`--workers N`、`--k 1 10`(human-eval pass@k)、 +`--eval-timeout S`、`--dataset-name`(SWE-bench 数据集名)。 + +退出码:`0` 通过 / `1` 未通过 / `2` 前置条件不满足(缺题库、缺 docker、byteseek 未就绪)/ `3` 参数错误。 + +## suite 与评测后端 + +| suite | 题量 | 评测 | +|-------|------|------| +| `humaneval` | 164 | 官方 `human_eval.evaluate_functional_correctness`(pass@k) | +| `humaneval+` | 164 | `evalplus.evaluate`(HumanEvalPlus,base / plus 两档) | +| `swebench-verified` | 500 | `swebench.harness.run_evaluation`(**需 Docker**) | +| `swebench-lite` | 300 | 同上 | + +agent 阶段与评测阶段分离,与官方 harness 一致:`run` 只产 predictions,`eval` 才判分。 +因此可以本机产 predictions、拿到有 Docker 的机器上判分。 + +## agent 契约 + +- **`gold`**:直接用题库自带答案(`canonical_solution` / `patch`),用来验证管道本身,不调 LLM。 +- **`byteseek`**:每题生成一个 `.kv`,内容为 `llm·call(<题目提示>) -> e` 加 `byteseek·run(e)`, + 由 `kvlang` 驱动并捕获 stdout。agent 必须把答案包在 `<<>>` 与 `<<>>` 之间; + 未取到标记时只在日志里提示,prediction 记为空串(不伪造内容)。 + + HumanEval 题问「补全该 Python 函数」;SWE-bench 题问「按问题描述产出 unified diff」。 + SWE-bench 默认只取 diff(不检出仓库);需要 agent 直接改仓库时,在提示里给出检出目录。 + +## 环境 + +重依赖装在 `BENCH_HOME`(默认 `/home/peng.li24/benchmarks`,跨 pod 持久),**不进 git 仓库**: + +``` +$BENCH_HOME/ + venv/ human-eval + evalplus + swebench + datasets + data/human-eval/ HumanEval.jsonl.gz(openai/human-eval 官方题) + data/evalplus/ HumanEvalPlus.jsonl(evalplus release) + data/swe-bench/ SWE-bench_Verified.jsonl / SWE-bench_Lite.jsonl(HF princeton-nlp) + runs/--<时间>/ tasks/ logs/ predictions.jsonl run.json + trash/ 旧 agentbench 题目(已被本入口取代) +``` + +重装: + +```bash +python3 -m venv $BENCH_HOME/venv +$BENCH_HOME/venv/bin/pip install human-eval evalplus swebench datasets huggingface_hub +``` + +## 已验证(2026-09-13) + +| 检查 | 结果 | |------|------| -| arithmetic | 数值计算与算术表达式 | -| string | 字符串拼接 / 反转 / 统计 | -| array | 数组算法(最值 / 求和 / 查找 / 排序) | -| dict | 键值结构与路径指针 | -| tool-shell | shell 工具调用 | -| tool-python | python 工具调用 | -| tool-http | http 网络抓取 | -| kvspace | 状态树读写与持久化 | -| reasoning | 逻辑推理与约束满足 | -| planning | 多步规划与多工具组合 | -| meta | 记忆、自省、自改(高阶) | - -## 答案约定 - -- 确定性题目:`.answer` 即 `println` 的期望输出(裸值,不含换行)。 -- 冒烟题目(需真实 LLM / 网络 / 完整 bootstrap):`.answer` 以 `期望(冒烟):` 开头,给出可验证检查。 -- 开放题目(难度 10):`.answer` 以 `评分标准:` 开头,给出 rubric,无单一标准输出。 - -## 环境假设 - -冒烟题依赖完整 byteseek runtime(已 bootstrap,`/lib/byteseek.kvlangbrief` 与 -`/byteseek/prompt/system` 已播种)、可用的 `DEEPSEEK_API_KEY`(嵌套代码生成题)与出网能力 -(http 题,走 `http_proxy`/`https_proxy` 环境变量)。 +| `run humaneval --agent gold` + `eval humaneval` | `pass@1 = 0.9939`(164 题,1 题官方 canonical 解过不了官方测试) | +| `eval humaneval+`(同一批 gold 样本) | `base 0.9878 / plus 0.8537`(canonical 解在 extra tests 上的已知表现) | +| `run swebench-lite --agent gold` | 产出 5 题标准格式 predictions | +| `eval swebench-lite` | 无 docker → 退出码 2,并打印可在有 Docker 机器直接执行的命令 | +| `run humaneval --agent byteseek --limit 20` + `eval` | **`pass@1 = 0.80`**(20 题,真 LLM,单条轨迹无重试) | + +## byteseek lib 与 kvlang 版本对齐(2026-09-14 已修) + +byteseek 现在只跟 kvlang 最新 release(`ci/deps.sh`,无 deps.json),lib 必须持续对齐 kvlang +语法。已修的三类问题: + +| 位置 | 旧 | 新 | +|------|-----|-----| +| `llm.kv` | `/tmp/esc·sys`(`·` 被当成员分隔符) | `/tmp/esc/sys` | +| `shell.kv` / `python.kv` | `noenv = map()`、`a = {"bash", …}`(无 langtype 的 `{}`) | `noenv:[int64]·[]char/utf32 = {}`、`a:[int64]·[]char/utf32 = {"bash", …}` | +| `llm.kv` / `memory.kv` / `memgen.kv` | `kv·get/set/has/listlen/listn` | `kvspace·get/set/has/listlen/listn` | +| `llm.kv` | `kvlanglayout·vet/layout` | `kvlang·vet/layout` | + +自检:`KVLANG_LIB=lib kvlang` 无 layout 报错,`/lib/{llm·call,shell·run,python·run}` 均在; +`shell·run` / `python·run` 冒烟输出正确;`--agent byteseek` 可跑完整链路。 diff --git a/agentbench/bench b/agentbench/bench new file mode 100755 index 0000000..5dc58ae --- /dev/null +++ b/agentbench/bench @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# agentbench 入口:所有重依赖(human-eval / evalplus / swebench / datasets)在 BENCH_HOME 的 venv 里。 +set -euo pipefail +BENCH_HOME=${BENCH_HOME:-/home/peng.li24/benchmarks} +PY=$BENCH_HOME/venv/bin/python +[ -x "$PY" ] || { echo "缺少 $PY(先按 agentbench/README.md 安装基准框架)" >&2; exit 2; } +exec "$PY" "$(dirname "$(readlink -f "$0")")/bench.py" "$@" diff --git a/agentbench/bench.py b/agentbench/bench.py new file mode 100755 index 0000000..ebd0caa --- /dev/null +++ b/agentbench/bench.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""agentbench —— HumanEval / SWE-bench 基准入口。 + + bench env 环境自检(框架 / 题库 / byteseek / docker) + bench list 列出 suite 与题量 + bench run [选项] 生成 predictions(agent 阶段) + bench eval [选项] 官方评测(eval 阶段) + bench all [选项] run + eval + +suite: humaneval | humaneval+ | swebench-verified | swebench-lite +agent: gold(题库自带答案,验管道) | byteseek(驱动 byteseek 代码脑) + +退出码:0 通过 / 1 未通过 / 2 前置条件不满足(缺题库、缺 docker、byteseek 不可用)/ 3 参数错误。 +""" + +import argparse +import gzip +import json +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +BENCH_HOME = Path(os.environ.get("BENCH_HOME", "/home/peng.li24/benchmarks")) +DATA = BENCH_HOME / "data" +RUNS = BENCH_HOME / "runs" +REPO = Path(__file__).resolve().parent.parent +KVLANG = os.environ.get("KVLANG", "kvlang") +KVSPACE = os.environ.get("KVSPACE", "redis://127.0.0.1:6379") + +BEGIN = "<<>>" +END = "<<>>" + +SUITES = { + "humaneval": {"kind": "humaneval", "file": DATA / "human-eval/HumanEval.jsonl.gz"}, + "humaneval+": {"kind": "humaneval", "file": DATA / "evalplus/HumanEvalPlus.jsonl"}, + "swebench-verified": {"kind": "swebench", "file": DATA / "swe-bench/SWE-bench_Verified.jsonl"}, + "swebench-lite": {"kind": "swebench", "file": DATA / "swe-bench/SWE-bench_Lite.jsonl"}, +} + + +# ── 题库 ──────────────────────────────────────────────────────────────── +def load_tasks(suite): + spec = SUITES[suite] + path = spec["file"] + if not path.exists(): + die(2, f"题库缺失:{path}(先跑 bench env 看安装状态)") + op = gzip.open if path.suffix == ".gz" else open + with op(path, "rt", encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def task_id_of(task): + return task.get("task_id") or task["instance_id"] + + +# ── agent 阶段 ────────────────────────────────────────────────────────── +def kv_literal(text): + """Python 串 → kvlang 双引号字面量(kvlang 与主流语言一致的 \\n \\" \\\\ 转义)。""" + return json.dumps(text, ensure_ascii=False) + + +def boot_byteseek(): + env = dict(os.environ, KVLANG_LIB="lib", KVSPACE=KVSPACE) + r = subprocess.run([KVLANG], cwd=REPO, env=env, capture_output=True, text=True) + if r.returncode != 0: + die(2, f"byteseek 引导失败:\n{r.stdout}{r.stderr}") + if not kvspace_get("/lib/llm·call/[0,0]"): + die(2, "byteseek 未就绪:/lib/llm·call 缺失——lib/llm.kv 与当前 kvlang 版本不兼容" + "(`KVLANG_LIB=lib kvlang` 的完整报错见上)。修好 lib 再跑 --agent byteseek。") + return r.stdout + r.stderr + + +def kvspace_get(key): + r = subprocess.run(["kvspace", "get", key], capture_output=True, text=True, + env=dict(os.environ, KVSPACE=KVSPACE)) + return "" if "(nil)" in r.stdout or r.returncode else r.stdout.strip() + + +def prompt_for(kind, task, repo_dir=None): + if kind == "humaneval": + return ( + "用 byteseek 完成下面的 Python 编程题:补全函数,把**完整可运行的 Python 实现**" + "打印出来(不要解释、不要额外文字)。\n" + f"输出必须严格包在 {BEGIN} 与 {END} 之间。\n\n" + f"题目:\n{task['prompt']}\n" + ) + where = f"代码仓库已检出在 {repo_dir},可用 shell 工具查阅与修改。\n" if repo_dir else "" + return ( + "用 byteseek 修复下面的软件缺陷,输出一个可 patch 的 unified diff" + "(`git diff` 格式,不要解释)。\n" + f"输出必须严格包在 {BEGIN} 与 {END} 之间。\n{where}\n" + f"仓库:{task['repo']}\n基线 commit:{task['base_commit']}\n" + f"问题描述:\n{task['problem_statement']}\n" + ) + + +def extract(text): + # 提示词与生成源码里都会回显标记(llm.print=1 会打印需求与生成代码), + # 所以取**最后**一段:那才是 agent 真正打印的答案。 + ms = re.findall(re.escape(BEGIN) + r"\s*(.*?)\s*" + re.escape(END), text, re.S) + return ms[-1] if ms else "" + + +def strip_prompt(block, prompt): + """agent 若把原题函数签名一起打印了,剥掉前缀,只留 completion。""" + head = prompt.strip().splitlines()[0].strip() if prompt.strip() else "" + pos = block.find(head) if head else -1 + return block[pos + len(head):] if pos >= 0 else block + + +def run_byteseek(kind, task, outdir, timeout): + tid = task_id_of(task).replace("/", "_") + tasks_dir = outdir / "tasks" + tasks_dir.mkdir(parents=True, exist_ok=True) + kvfile = tasks_dir / f"{tid}.kv" + kvfile.write_text( + "rwfunc test() -> () {\n" + f"\tllm·call({kv_literal(prompt_for(kind, task))}) -> e\n" + "\tbyteseek·run(e)\n" + "}\n", + encoding="utf-8", + ) + env = dict(os.environ, KVSPACE=KVSPACE) + try: + r = subprocess.run([KVLANG, str(kvfile)], cwd=REPO, env=env, + capture_output=True, text=True, timeout=timeout) + raw = r.stdout + r.stderr + except subprocess.TimeoutExpired as e: + raw = (e.stdout or "") + (e.stderr or "") + f"\n[bench] 超时 {timeout}s" + (outdir / "logs").mkdir(parents=True, exist_ok=True) + (outdir / "logs" / f"{tid}.log").write_text(raw, encoding="utf-8") + return extract(raw), raw + + +def predict_gold(kind, task): + if kind == "humaneval": + sol = task["canonical_solution"] + return sol if task.get("prompt", "") in sol else task["prompt"] + sol + return task["patch"] + + +def cmd_run(a): + kind = SUITES[a.suite]["kind"] + tasks = load_tasks(a.suite) + if a.task: + want = set(a.task.split(",")) + tasks = [t for t in tasks if task_id_of(t) in want] + tasks = tasks[a.offset:] + if a.limit: + tasks = tasks[: a.limit] + if not tasks: + die(3, "没有选中任何题目") + + outdir = Path(a.out) if a.out else RUNS / f"{a.suite}-{a.agent}-{time.strftime('%Y%m%d-%H%M%S')}" + outdir.mkdir(parents=True, exist_ok=True) + if a.agent == "byteseek" and not a.no_boot: + print("[bench] 引导 byteseek ...", file=sys.stderr) + boot_byteseek() + + preds = [] + for i, task in enumerate(tasks, 1): + tid = task_id_of(task) + if a.agent == "gold": + ans = predict_gold(kind, task) + else: + ans, raw = run_byteseek(kind, task, outdir, a.timeout) + if not ans: + print(f"[bench] {i}/{len(tasks)} {tid}: 未取到答案标记,原始输出见 logs/", file=sys.stderr) + if kind == "humaneval": + completion = strip_prompt(ans, task["prompt"]) + preds.append({"task_id": tid, "completion": completion, "solution": task["prompt"] + completion}) + else: + preds.append({"instance_id": tid, "model_patch": ans, "model_name_or_path": f"byteseek-{a.agent}"}) + print(f"[bench] {i}/{len(tasks)} {tid}: {len(ans)} 字符", file=sys.stderr) + + pred_file = outdir / "predictions.jsonl" + with pred_file.open("w", encoding="utf-8") as fh: + for p in preds: + fh.write(json.dumps(p, ensure_ascii=False) + "\n") + (outdir / "run.json").write_text( + json.dumps({"suite": a.suite, "agent": a.agent, "n": len(preds), + "predictions": str(pred_file)}, ensure_ascii=False, indent=1)) + print(pred_file) + return 0 + + +# ── eval 阶段 ─────────────────────────────────────────────────────────── +def cmd_eval(a): + kind = SUITES[a.suite]["kind"] + pred = Path(a.predictions) if a.predictions else latest_predictions(a.suite) + if not pred.exists(): + die(2, f"predictions 不存在:{pred}(先 bench run)") + if kind == "humaneval": + return eval_humaneval(a.suite, pred, a) + return eval_swebench(a.suite, pred, a) + + +def latest_predictions(suite): + cands = sorted(RUNS.glob(f"{suite}-*/predictions.jsonl"), key=lambda p: p.stat().st_mtime) + return cands[-1] if cands else RUNS / f"{suite}-none/predictions.jsonl" + + +def eval_humaneval(suite, pred, a): + if suite == "humaneval": + from human_eval.evaluation import evaluate_functional_correctness + res = evaluate_functional_correctness( + str(pred), problem_file=problems_for(suite, pred), k=a.k, + n_workers=a.workers, timeout=a.eval_timeout) + print(json.dumps(res, ensure_ascii=False)) + return 0 if res.get(f"pass@{a.k[0]}", 0) > 0 else 1 + need = {t["task_id"] for t in load_tasks(suite)} + have = {json.loads(l)["task_id"] for l in pred.open(encoding="utf-8") if l.strip()} + if have != need: + die(2, f"evalplus 要求样本覆盖全部 {len(need)} 题,当前只有 {len(have & need)} 题" + f"(先 bench run {suite} --agent ... 跑全量)") + r = subprocess.run([sys.executable, "-m", "evalplus.evaluate", "--samples", str(pred), + "--dataset", "humaneval", "--parallel", str(a.workers)], + capture_output=True, text=True) + if r.returncode != 0: + print(r.stdout + r.stderr) + return r.returncode + res = Path(str(pred).replace(".jsonl", "_eval_results.json")) + if res.exists(): + ev = json.loads(res.read_text())["eval"] + n = len(ev) + base = sum(1 for v in ev.values() if v[0]["base_status"] == "pass") + plus = sum(1 for v in ev.values() if v[0]["plus_status"] == "pass") + print(json.dumps({"n": n, "base_pass@1": round(base / n, 4), "plus_pass@1": round(plus / n, 4), + "results": str(res)}, ensure_ascii=False)) + return 0 if plus > 0 else 1 + print(r.stdout + r.stderr) + return r.returncode + + +def problems_for(suite, pred): + """human-eval 要求样本覆盖题库全部题目;跑子集时裁一份同名题库陪跑。""" + src = SUITES[suite]["file"] + ids = {json.loads(l)["task_id"] for l in pred.open(encoding="utf-8") if l.strip()} + rows = load_tasks(suite) + if {r["task_id"] for r in rows} == ids: + return str(src) + out = pred.parent / "problems.jsonl" + with out.open("w", encoding="utf-8") as fh: + for r in rows: + if r["task_id"] in ids: + fh.write(json.dumps(r, ensure_ascii=False) + "\n") + return str(out) + + +def eval_swebench(suite, pred, a): + if not shutil.which("docker"): + print( + "SWE-bench 官方评测需要 Docker(每个实例在自己的容器里装环境、跑测试)。\n" + f"当前环境没有 docker。predictions 已就绪,可在有 Docker 的机器上执行:\n\n" + f" python -m swebench.harness.run_evaluation \\\n" + f" --dataset_name {a.dataset_name or 'princeton-nlp/SWE-bench_Lite'} \\\n" + f" --predictions_path {pred} --max_workers {a.workers} \\\n" + f" --run_id bench-{time.strftime('%Y%m%d-%H%M%S')} --cache_level env\n\n" + "或只做「能跑通」的轻量验证:byteseek 生成的 patch 非空即视为产出成功。", + file=sys.stderr) + return 2 + from swebench.harness.run_evaluation import main as run_evaluation + run_id = f"bench-{time.strftime('%Y%m%d-%H%M%S')}" + sys.argv = ["run_evaluation", "--dataset_name", a.dataset_name or "princeton-nlp/SWE-bench_Lite", + "--predictions_path", str(pred), "--max_workers", str(a.workers), + "--run_id", run_id, "--cache_level", "env"] + run_evaluation() + return 0 + + +# ── 自检 ──────────────────────────────────────────────────────────────── +def cmd_env(_a): + def ver(mod): + try: + __import__(mod) + except Exception as e: + return f"缺失({e.__class__.__name__})" + try: + from importlib.metadata import version + return version(mod) + except Exception: + return "已装(版本未知)" + + print(f"BENCH_HOME : {BENCH_HOME}") + print(f"python : {sys.version.split()[0]} ({sys.executable})") + print(f"frameworks : human-eval {ver('human_eval')} | evalplus {ver('evalplus')} | " + f"swebench {ver('swebench')} | datasets {ver('datasets')}") + print(f"docker : {shutil.which('docker') or '缺失(SWE-bench 官方评测不可用)'}") + print(f"kvlang : {shutil.which(KVLANG) or '缺失'} KVSPACE={KVSPACE}") + print(f"byteseek : {REPO}") + for suite in SUITES: + path = SUITES[suite]["file"] + n = len(load_tasks(suite)) if path.exists() else 0 + print(f" {suite:<18} {n:>4} 题 {path if path.exists() else '缺失'}") + for key in ("/byteseek/llm.key", "/byteseek/prompt/system"): + r = subprocess.run(["kvspace", "get", key], capture_output=True, text=True, + env=dict(os.environ, KVSPACE=KVSPACE)) + ok = "(nil)" not in r.stdout + print(f"kvspace {key:<22} {'已种入' if ok else '未种入(先 KVLANG_LIB=lib kvlang)'}") + return 0 + + +def cmd_list(_a): + for suite, spec in SUITES.items(): + n = len(load_tasks(suite)) if spec["file"].exists() else 0 + print(f"{suite:<18} {spec['kind']:<10} {n:>4} 题") + return 0 + + +def die(code, msg): + print(f"[bench] {msg}", file=sys.stderr) + sys.exit(code) + + +def main(): + ap = argparse.ArgumentParser(prog="bench", description="HumanEval / SWE-bench 基准入口") + sub = ap.add_subparsers(dest="cmd", required=True) + + sub.add_parser("env", help="环境自检").set_defaults(fn=cmd_env) + sub.add_parser("list", help="列出 suite 与题量").set_defaults(fn=cmd_list) + + def add_common(p): + p.add_argument("suite", choices=list(SUITES)) + p.add_argument("--predictions", help="predictions.jsonl(默认取最近一次 run)") + p.add_argument("--workers", type=int, default=8) + p.add_argument("--timeout", type=int, default=300) + + r = sub.add_parser("run", help="生成 predictions") + add_common(r) + r.add_argument("--agent", choices=["gold", "byteseek"], default="byteseek") + r.add_argument("--limit", type=int, default=0, help="只跑前 N 题(0 = 全部)") + r.add_argument("--offset", type=int, default=0) + r.add_argument("--task", help="逗号分隔的 task_id / instance_id 白名单") + r.add_argument("--out", help="输出目录(默认 $BENCH_HOME/runs/--<时间>)") + r.add_argument("--no-boot", action="store_true", help="跳过 byteseek 引导") + r.set_defaults(fn=cmd_run) + + def add_eval_args(p): + p.add_argument("--k", type=int, nargs="+", default=[1], help="pass@k(humaneval)") + p.add_argument("--eval-timeout", type=float, default=10.0, help="单题执行超时秒数") + p.add_argument("--dataset-name", help="SWE-bench 数据集名(默认 SWE-bench_Lite)") + + e = sub.add_parser("eval", help="官方评测") + add_common(e) + add_eval_args(e) + e.set_defaults(fn=cmd_eval) + + x = sub.add_parser("all", help="run + eval") + add_common(x) + add_eval_args(x) + x.add_argument("--agent", choices=["gold", "byteseek"], default="byteseek") + x.add_argument("--limit", type=int, default=0) + x.add_argument("--task") + x.set_defaults(fn=None) + + a = ap.parse_args() + if a.cmd == "all": + rc = cmd_run(argparse.Namespace(**{**vars(a), "offset": 0, "out": None, "no_boot": False})) + if rc: + return rc + return cmd_eval(a) + return a.fn(a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/deps.sh b/ci/deps.sh index 17424a2..fb164b0 100755 --- a/ci/deps.sh +++ b/ci/deps.sh @@ -1,19 +1,40 @@ #!/usr/bin/env bash -# 下载 ABI 依赖(deps.json: repo → tag),安装到 /usr/lib + /usr/include。本地与 CI 共用。 +# byteseek 的唯一依赖 = kvlang。不再有 deps.json:kvlang 的 release 是自包含的—— +# bin/{kvlang,kvlanglayout,kvspace} + lib/ + lib/kvspace/(两个后端)+ include/, +# 一次装齐 /usr,本地与 CI 共用。 +# +# KVLANG_VERSION= 钉版本(如 v0.2.18);默认取**最新** release。 +# +# 版本号解析走带 token 的 GitHub API(GITHUB_TOKEN / GH_TOKEN / `gh auth token`)—— +# 匿名 api.github.com 常被限流成 403,而 release 包本身是公开的、下载不需要 token。 set -euo pipefail -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -for repo in $(jq -r 'keys[]' "$ROOT/deps.json"); do - ver=$(jq -r ".\"$repo\"" "$ROOT/deps.json") - tmp="$(mktemp -d)" - gh release download "$ver" -R "array2d/$repo" -p "${repo}-abi-*-linux-x86_64.tar.gz" -D "$tmp" - tar xzf "$tmp"/*.tar.gz -C "$tmp" --strip-components=1 - if [ -d "$tmp/include" ]; then sudo cp -r "$tmp/include/"* /usr/include/; fi - if [ "$repo" = "kvspace-c" ] || [ "$repo" = "kvspace-durable" ]; then - sudo mkdir -p /usr/lib/kvspace - sudo cp "$tmp/lib/"*.so* /usr/lib/kvspace/ - else - sudo cp "$tmp/lib/"*.so* /usr/lib/ + +URL=${KVLANG_INSTALL_URL:-https://raw.githubusercontent.com/array2d/kvlang/master/install.sh} +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +curl -fsSL "$URL" -o "$tmp/install.sh" + +tag="${KVLANG_VERSION:-}" +if [ -z "$tag" ]; then + token="${GITHUB_TOKEN:-${GH_TOKEN:-}}" + if [ -z "$token" ] && command -v gh >/dev/null 2>&1; then + token=$(gh auth token 2>/dev/null || true) fi - rm -rf "$tmp" -done -echo "✅ ABI deps → /usr/lib + /usr/include" + if [ -n "$token" ]; then + tag=$(curl -fsSL -H "Authorization: Bearer $token" \ + https://api.github.com/repos/array2d/kvlang/releases/latest | + sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) + fi +fi + +# 平台识别 + sha256 校验 + 安装都在 install.sh 里;tag 为空时它自己取最新 +VERSION="$tag" sh "$tmp/install.sh" + +# install.sh 把库放 /lib/kvspace,但 kvspace CLI 以 libkvspace.so.1 直接链接该目录, +# 而它不在默认加载路径里(kvlang/kvlanglayout 自带 rpath,kvspace 没有)→ 补 loader 路径。 +if [ "$(uname -s)" = Linux ]; then + prefix="${PREFIX:-/usr}" + echo "$prefix/lib/kvspace" | sudo tee /etc/ld.so.conf.d/kvspace.conf >/dev/null + sudo ldconfig +fi diff --git a/deps.json b/deps.json deleted file mode 100644 index bc75fe5..0000000 --- a/deps.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "kvlang": "v0.2.15", - "kvspace": "v0.2.15", - "kvspace-durable": "v0.2.15", - "kvspace-c": "v0.2.15", - "blockmalloc": "v0.1.4", - "slotsboxmalloc": "v0.1.5" -} diff --git a/doc/jit-agent-reference.md b/doc/jit-agent-reference.md index e5845d8..e003de7 100644 --- a/doc/jit-agent-reference.md +++ b/doc/jit-agent-reference.md @@ -82,7 +82,7 @@ agent 程序(`harness_ops.py` 解析模型的 5 个 tagged block)。 各成一个 `.kv` 库,`llm·call` 生成时按四模块组装,而非自由生成。 2. **补 best-of-N 择优**:#7 的「n 变体实测择优」可直接借鉴 `selector.py`——生成 n 个 候选 harness,跑完按 logprob 或 judge 评分,沉淀最优到 `/lib/byteseek/session/` 档案库。 -3. **generate→validate→repair 循环**:byteseek 已有 `kvlanglayout·vet` 闸门,可扩成 +3. **generate→validate→repair 循环**:byteseek 已有 `kvlang·vet` 闸门,可扩成 JIT 式完整循环(生成 → vet/试跑 → 反馈 → repair)。 4. **记忆模块对照**:#9/#10/#11 的实现可对照 `memobrain`(依赖感知推理图 + token 预算压缩) 与 `resum`(摘要),选定一种在 kvspace 里落地。 diff --git a/doc/substrate.md b/doc/substrate.md index 3d2b3ca..b4f3658 100644 --- a/doc/substrate.md +++ b/doc/substrate.md @@ -79,8 +79,8 @@ rwfunc mainbrain() -> () { ``` `llm·call(userinput)`:system prompt(提示词 + 语法速览)→ LLM → 解析 ``/`` → -包进 `lib byteseek { lib session { lib NAME { … NAME·main() } } }` → `kvlanglayout·vet` 校验 → -通过后 `kvlanglayout·layout` 入库 → 返回入口 `byteseek/session/NAME·init`。生成失败回填 +包进 `lib byteseek { lib session { lib NAME { … NAME·main() } } }` → `kvlang·vet` 校验 → +通过后 `kvlang·layout` 入库 → 返回入口 `byteseek/session/NAME·init`。生成失败回填 `error: …`,`byteseek·run` 据前缀跳过执行。 ## byteseek·run:同 vthread 动态执行(进程↔vid 1:1) @@ -109,7 +109,7 @@ byteseek 不再注册任何自有 Rust rwir。所需能力全部是 kvlang 标 | `print` / `println` / `cerr` / `input` | 标准 term rwir | | `json·to` / `json·from` | 标准 json rwir | | `http·call(method,header,url,body) -> resp` | 标准 http rwir | -| `kvlanglayout·vet / ·format / ·layout / ·dump` | 标准 layout rwir | +| `kvlang·vet / ·format / ·layout / ·dump` | 标准 layout rwir | | `networld/proc·exec(args,envs) -> code, out, err` | 标准 networld rwir(子进程 + 捕获 @ 句柄) | | `vthread·call(funckey)` | native builtin(同 vid 动态调用) | | `string·* / kv·* / xv·*` | native builtin | diff --git a/lib/byteseek/llm.kv b/lib/byteseek/llm.kv index a2a702b..f4bbf93 100644 --- a/lib/byteseek/llm.kv +++ b/lib/byteseek/llm.kv @@ -4,22 +4,22 @@ lib llm { rwfunc call(userinput:[]char/utf32) -> (entry:[]char/utf32) { - kv·get("/byteseek/llm.key") -> key + kvspace·get("/byteseek/llm.key") -> key if (key == "") { "error: 未设置 DEEPSEEK_API_KEY" -> entry return } - kv·get("/byteseek/llm.api") -> url - kv·get("/byteseek/llm.print") -> print + kvspace·get("/byteseek/llm.api") -> url + kvspace·get("/byteseek/llm.print") -> print if (print != "0") { println("[llm] 需求:", userinput) } - kv·get("/byteseek/prompt/system") -> sys - kv·get("/lib/kvlang/kvlangbrief") -> brief + kvspace·get("/byteseek/prompt/system") -> sys + kvspace·get("/lib/kvlang/kvlangbrief") -> brief sys + "\n\n=== kvlang 语法速览 ===\n" + brief -> sys "请把下面的需求翻译成一段 kvlang 程序。严格按系统提示的格式输出,只输出 / 两段,不要解释:\n\n" + userinput -> user - sys -> /tmp/esc·sys - json·to(/tmp/esc·sys) -> sys_j - user -> /tmp/esc·user - json·to(/tmp/esc·user) -> user_j + sys -> /tmp/esc/sys + json·to(/tmp/esc/sys) -> sys_j + user -> /tmp/esc/user + json·to(/tmp/esc/user) -> user_j "{\"model\":\"deepseek-v4-flash\",\"messages\":[{\"role\":\"system\",\"content\":" + sys_j + "},{\"role\":\"user\",\"content\":" + user_j + "}],\"temperature\":0.2,\"top_p\":1.0,\"max_tokens\":4096,\"stream\":false}" -> body "Content-Type: application/json\nAuthorization: Bearer " + key -> header http·call("POST", header, url, body) -> resp @@ -29,7 +29,7 @@ lib llm { return } json·from(resp) -> /tmp/llmresp - kv·get("/tmp/llmresp·choices·[0]·message·content") -> content + kvspace·get("/tmp/llmresp·choices·[0]·message·content") -> content if (content == "") { "error: LLM 响应无内容: " + resp -> entry return @@ -56,12 +56,12 @@ lib llm { println(kv) } "lib byteseek {\nlib session {\nlib " + name + " {\n" + kv + "\nbyteseek/session/" + name + "·main()\n}\n}\n}\n" -> wrapped - kvlanglayout·vet(wrapped) -> v + kvlang·vet(wrapped) -> v if (v != "ok") { "error: vet 失败: " + v -> entry return } - kvlanglayout·layout(wrapped) -> e2 + kvlang·layout(wrapped) -> e2 string·find(e2, "error") -> has_err if (has_err >= 0) { e2 -> entry @@ -74,16 +74,16 @@ lib llm { // llm·raw(sys, user) -> text:裸 LLM 调用,返回 assistant 文本(不生成/不执行 kv 代码)。 // 供 memgen 等需要「LLM 直接产出一段文本」的元能力复用。 rwfunc raw(sys:[]char/utf32, user:[]char/utf32) -> (text:[]char/utf32) { - kv·get("/byteseek/llm.key") -> key + kvspace·get("/byteseek/llm.key") -> key if (key == "") { "error: 未设置 DEEPSEEK_API_KEY" -> text return } - kv·get("/byteseek/llm.api") -> url - sys -> /tmp/esc·sys - json·to(/tmp/esc·sys) -> sys_j - user -> /tmp/esc·user - json·to(/tmp/esc·user) -> user_j + kvspace·get("/byteseek/llm.api") -> url + sys -> /tmp/esc/sys + json·to(/tmp/esc/sys) -> sys_j + user -> /tmp/esc/user + json·to(/tmp/esc/user) -> user_j "{\"model\":\"deepseek-v4-flash\",\"messages\":[{\"role\":\"system\",\"content\":" + sys_j + "},{\"role\":\"user\",\"content\":" + user_j + "}],\"temperature\":0.2,\"top_p\":1.0,\"max_tokens\":1024,\"stream\":false}" -> body "Content-Type: application/json\nAuthorization: Bearer " + key -> header http·call("POST", header, url, body) -> resp @@ -93,7 +93,7 @@ lib llm { return } json·from(resp) -> /tmp/llmresp - kv·get("/tmp/llmresp·choices·[0]·message·content") -> content + kvspace·get("/tmp/llmresp·choices·[0]·message·content") -> content if (content == "") { "error: LLM 响应无内容: " + resp -> text return diff --git a/lib/byteseek/memgen.kv b/lib/byteseek/memgen.kv index 2247595..e127e5d 100644 --- a/lib/byteseek/memgen.kv +++ b/lib/byteseek/memgen.kv @@ -8,10 +8,10 @@ lib byteseek { rwfunc fnlist(name:[]char/utf32) -> (listing:[]char/utf32) { "" -> listing "/lib/" + name + "·" -> mdir - kv·listlen(mdir) -> n + kvspace·listlen(mdir) -> n i = 0 while (i < n) { - kv·listn(mdir, i) -> entry + kvspace·listn(mdir, i) -> entry char/utf32(entry) -> entry string·len(entry) -> elen string·find(entry, ".src") -> p @@ -31,7 +31,7 @@ lib byteseek { } rwfunc summarize(name:[]char/utf32) -> () { - kv·has("/lib/" + name + ".mem") -> exists + kvspace·has("/lib/" + name + ".mem") -> exists if (exists == true) { return } fnlist(name) -> listing "你是 kvlang 能力记忆生成器。kvspace 目录 /lib/" + name + " 提供以下能力:" + listing + "。请用中文写一段不超过 200 字的紧凑能力记忆:用 · 分隔,每个能力名后跟一句话用途。只输出记忆文本本身,不要代码、不要解释、不要 markdown。" -> user @@ -41,7 +41,7 @@ lib byteseek { println("[memgen] 概括失败:", name, text) return } - kv·set("/lib/" + name + ".mem", text) + kvspace·set("/lib/" + name + ".mem", text) println("[memgen] 已生成:", "/lib/" + name + ".mem") } diff --git a/lib/byteseek/memory.kv b/lib/byteseek/memory.kv index 7a13ae7..b657f53 100644 --- a/lib/byteseek/memory.kv +++ b/lib/byteseek/memory.kv @@ -9,19 +9,19 @@ lib byteseek { lib memory { rwfunc remember(key:[]char/utf32, value:[]char/utf32) -> () { "/byteseek/memory/" + key -> path - kv·set(path, value) + kvspace·set(path, value) } rwfunc recall(query:[]char/utf32) -> (ctx:[]char/utf32) { "" -> ctx - kv·listlen("/byteseek/memory/") -> n + kvspace·listlen("/byteseek/memory/") -> n i = 0 while (i < n) { - kv·listn("/byteseek/memory/", i) -> key + kvspace·listn("/byteseek/memory/", i) -> key char/utf32(key) -> key if (key != "cat") { "/byteseek/memory/" + key -> path - kv·get(path) -> value + kvspace·get(path) -> value hit = 0 string·find(query, key) -> a if (a != -1) { hit = 1 } @@ -57,9 +57,9 @@ lib byteseek { rwfunc ensure_category(cat:[]char/utf32) -> () { if (cat == "") { return } "/byteseek/memory/cat/" + cat -> path - kv·has(path) -> exists + kvspace·has(path) -> exists if (exists == false) { - kv·set(path, "1") + kvspace·set(path, "1") println("[memory] 新类别:", cat) } } diff --git a/lib/byteseek/python.kv b/lib/byteseek/python.kv index f593def..4b1738b 100644 --- a/lib/byteseek/python.kv +++ b/lib/byteseek/python.kv @@ -3,8 +3,8 @@ lib python { rwfunc run(code:[]char/utf32) -> (out:[]char/utf32) { println("🔧 python:", code) - noenv = map() - a = {"python3", "-c", code} + noenv:[int64]·[]char/utf32 = {} + a:[int64]·[]char/utf32 = {"python3", "-c", code} networld/proc·exec(a, noenv) -> ret, o, e println("↳", o, e) o -> out diff --git a/lib/byteseek/shell.kv b/lib/byteseek/shell.kv index 8fd7e9d..249dc49 100644 --- a/lib/byteseek/shell.kv +++ b/lib/byteseek/shell.kv @@ -4,8 +4,8 @@ lib shell { rwfunc run(cmd:[]char/utf32) -> (out:[]char/utf32) { println("🔧 shell:", cmd) - noenv = map() - a = {"bash", "-c", cmd} + noenv:[int64]·[]char/utf32 = {} + a:[int64]·[]char/utf32 = {"bash", "-c", cmd} networld/proc·exec(a, noenv) -> code, o, e println("↳", o, e) o -> out