diff --git a/docs/superpowers/plans/2026-09-19-muse-106-sweep.md b/docs/superpowers/plans/2026-09-19-muse-106-sweep.md new file mode 100644 index 000000000..16ab6814d --- /dev/null +++ b/docs/superpowers/plans/2026-09-19-muse-106-sweep.md @@ -0,0 +1,2692 @@ +# MUSE 106-Case Sweep Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a resumable, parallel 106-case MUSE benchmark sweep driven by `deepseek-ai/DeepSeek-V4.1-Flash` through the kernelCAD agent stack, producing an official-submission-grade leaderboard package. + +**Architecture:** A batch runner (`scripts/runMuseSweep.ts`) drives the existing eval pipeline (`generateCase` → `scoreCase` from a split `eval/runner.ts`) over a promise pool; scoring and judging call MUSE's own code through `eval/oracle/museScorer.ts` + a new judge wrapper; a report stage aggregates into leaderboard artifacts. + +**Tech Stack:** TypeScript/tsx, vitest, Node 22 global fetch, Python 3.12 (MUSE venv: cadquery + vtk), DeepInfra OpenAI-compatible API. + +**Spec:** `docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md` + +--- + +## File Structure + +| File | Action | Responsibility | +|---|---|---| +| `eval/agentOpenAICompat.ts` | Create | OpenAI-compatible AgentClient (DeepInfra) | +| `eval/lib/systemPrompt.ts` | Create | Skill-prompt builder with explicit skill selection | +| `eval/lib/pool.ts` | Create | Bounded-concurrency map | +| `eval/lib/museState.ts` | Create | Per-case phase state machine + atomic writes | +| `eval/lib/museAggregate.ts` | Create | MUSE pillar arithmetic (pure) | +| `eval/runner.ts` | Modify | Split `runTask` into `generateCase` + `scoreCase` | +| `eval/oracle/museJudgeWrapper.py` | Create | One-sample judge via MUSE's own functions | +| `scripts/museJudge.ts` | Create | `judgeCase()` TS wrapper + standalone CLI | +| `scripts/musePreflight.ts` | Create | Env/asset verification | +| `scripts/runMuseSweep.ts` | Create | Batch CLI (pool, resume, budget, progress) | +| `scripts/museReport.ts` | Create | leaderboard.json/csv, summary.md, protocol.md | + +Tests live next to sources per repo convention: `eval/*.test.ts`, `eval/lib/*.test.ts`, `scripts/*.test.ts`. + +--- + +### Task 0: Worktree, prerequisites, import all 106 MUSE tasks + +**Files:** +- Create: `eval/tasks/muse-/` × 96 (generated by the importer; committed) + +- [ ] **Step 1: Create the worktree off develop** + +```bash +cd /home/andrii/projects/kernelCAD-web +git fetch origin +git worktree add ../kernelCAD-web-worktrees/muse-106-sweep -b feat/muse-106-sweep origin/develop +cd ../kernelCAD-web-worktrees/muse-106-sweep +npm install +npm run build:cli +``` + +Expected: `dist/cli/index.js` exists. + +- [ ] **Step 2: Set up the MUSE Python venv** + +```bash +cd /home/andrii/projects/muse +uv venv .venv --python 3.12 +uv pip install --python .venv/bin/python -e . +uv pip install --python .venv/bin/python requests +.venv/bin/python -c "import cadquery, vtk, requests; print('muse-env-ok')" +``` + +Expected: `muse-env-ok` (cadquery + vtk are the heavy installs, ~5 min). + +- [ ] **Step 3: Import all 106 tasks** + +Run from the worktree root: + +```bash +CASES=$(ls /home/andrii/projects/muse/data/muse/cases) +npx tsx eval/lib/importMuseTasks.ts --dataset /home/andrii/projects/muse/data/muse/cases --cases $CASES +ls eval/tasks | grep -c '^muse-' +``` + +Expected: `106`. + +- [ ] **Step 4: Verify importer idempotence** + +```bash +git status --porcelain eval/tasks | wc -l # note the count +npx tsx eval/lib/importMuseTasks.ts --dataset /home/andrii/projects/muse/data/muse/cases --cases $CASES +git status --porcelain eval/tasks | wc -l # must equal the count above +``` + +Expected: equal counts (re-run changes nothing). + +- [ ] **Step 5: Commit** + +```bash +git add eval/tasks +git commit -m "eval(muse): import all 106 MUSE cases as tasks" +``` + +--- + +### Task 1: OpenAI-compatible agent client + +**Files:** +- Create: `eval/agentOpenAICompat.ts` +- Test: `eval/agentOpenAICompat.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// eval/agentOpenAICompat.test.ts +import { describe, expect, it, vi } from 'vitest'; +import { OpenAICompatAgentClient } from './agentOpenAICompat'; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +const REQ = { + system: 'S', + messages: [{ role: 'user' as const, content: 'hi' }], + model: 'deepseek-ai/DeepSeek-V4.1-Flash', + max_tokens: 100, + temperature: 0.2, +}; + +describe('OpenAICompatAgentClient', () => { + it('returns text and usage from a successful completion', async () => { + const fetchImpl = vi.fn(async () => + jsonResponse({ + choices: [{ message: { content: 'hello' } }], + usage: { prompt_tokens: 11, completion_tokens: 7 }, + }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1/', + apiKey: 'k', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out).toEqual({ text: 'hello', tokens_in: 11, tokens_out: 7 }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe('https://api.example.com/v1/chat/completions'); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.model).toBe('deepseek-ai/DeepSeek-V4.1-Flash'); + expect(body.messages[0]).toEqual({ role: 'system', content: 'S' }); + expect(body.temperature).toBe(0.2); + }); + + it('retries 429 then succeeds', async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response('rate', { status: 429 })) + .mockResolvedValueOnce( + jsonResponse({ choices: [{ message: { content: 'ok' } }], usage: {} }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out.text).toBe('ok'); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it('retries a network error up to the cap, then throws', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + maxRetries: 2, + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + await expect(client.generate(REQ)).rejects.toThrow('ECONNRESET'); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it('throws on a non-retryable 400 without retrying', async () => { + const fetchImpl = vi.fn(async () => new Response('bad model', { status: 400 })); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + await expect(client.generate(REQ)).rejects.toThrow('HTTP 400'); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run eval/agentOpenAICompat.test.ts` +Expected: FAIL — cannot resolve `./agentOpenAICompat`. + +- [ ] **Step 3: Implement the client** + +```ts +// eval/agentOpenAICompat.ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import type { AgentClient, AgentMessage, AgentResponse } from './types'; + +export interface OpenAICompatOptions { + baseUrl: string; + apiKey: string; + /** Total attempts = maxRetries + 1. Default 5 (6 attempts). */ + maxRetries?: number; + retryBaseMs?: number; + retryMaxMs?: number; + fetchImpl?: typeof fetch; +} + +interface ChatCompletionResponse { + choices?: Array<{ message?: { content?: string | null } }>; + usage?: { prompt_tokens?: number; completion_tokens?: number }; +} + +const RETRYABLE_STATUS = (status: number): boolean => status === 429 || status >= 500; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export class OpenAICompatAgentClient implements AgentClient { + private readonly baseUrl: string; + private readonly apiKey: string; + private readonly maxRetries: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly fetchImpl: typeof fetch; + + constructor(opts: OpenAICompatOptions) { + this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); + this.apiKey = opts.apiKey; + this.maxRetries = opts.maxRetries ?? 5; + this.retryBaseMs = opts.retryBaseMs ?? 1000; + this.retryMaxMs = opts.retryMaxMs ?? 30000; + this.fetchImpl = opts.fetchImpl ?? fetch; + } + + async generate(args: { + system: string; + systemAddendum?: string; + messages: AgentMessage[]; + model: string; + max_tokens: number; + temperature?: number; + }): Promise { + const system = + args.systemAddendum && args.systemAddendum.length > 0 + ? `${args.system}\n\n${args.systemAddendum}` + : args.system; + const body = { + model: args.model, + max_tokens: args.max_tokens, + messages: [ + { role: 'system', content: system }, + ...args.messages.map((m) => ({ role: m.role, content: m.content })), + ], + ...(args.temperature !== undefined ? { temperature: args.temperature } : {}), + }; + + let lastErr: Error | undefined; + for (let attempt = 0; attempt <= this.maxRetries; attempt++) { + if (attempt > 0) { + const backoff = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** (attempt - 1)); + await sleep(backoff + Math.random() * backoff * 0.25); + } + let resp: Response; + try { + resp = await this.fetchImpl(`${this.baseUrl}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(180_000), + }); + } catch (err) { + lastErr = err instanceof Error ? err : new Error(String(err)); + continue; + } + if (RETRYABLE_STATUS(resp.status)) { + lastErr = new Error(`HTTP ${resp.status}`); + continue; + } + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + throw new Error(`OpenAI-compat request failed: HTTP ${resp.status} ${text.slice(0, 300)}`); + } + let data: ChatCompletionResponse; + try { + data = (await resp.json()) as ChatCompletionResponse; + } catch (err) { + lastErr = err instanceof Error ? err : new Error(String(err)); + continue; + } + const text = data.choices?.[0]?.message?.content ?? ''; + if (text.length === 0) { + return { text: '', tokens_in: data.usage?.prompt_tokens ?? 0, tokens_out: 0 }; + } + return { + text, + tokens_in: data.usage?.prompt_tokens ?? 0, + tokens_out: data.usage?.completion_tokens ?? 0, + }; + } + throw lastErr ?? new Error('OpenAI-compat request failed after retries'); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run eval/agentOpenAICompat.test.ts` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add eval/agentOpenAICompat.ts eval/agentOpenAICompat.test.ts +git commit -m "eval: add OpenAI-compatible agent client for DeepInfra sweep runs" +``` + +--- + +### Task 2: System-prompt builder + +**Files:** +- Create: `eval/lib/systemPrompt.ts` +- Test: `eval/lib/systemPrompt.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// eval/lib/systemPrompt.test.ts +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { buildSystemPrompt, SWEEP_SKILLS } from './systemPrompt'; + +describe('buildSystemPrompt', () => { + it('joins selected skills in sorted order with separators', () => { + const root = mkdtempSync(join(tmpdir(), 'skills-')); + for (const name of ['b', 'a']) { + mkdirSync(join(root, name)); + writeFileSync(join(root, name, 'SKILL.md'), `# ${name}`); + } + const out = buildSystemPrompt(['b', 'a'], root); + expect(out).toBe('# a\n\n---\n\n# b'); + }); + + it('throws when a selected skill is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'skills-')); + expect(() => buildSystemPrompt(['nope'], root)).toThrow("SKILL.md not found for skill 'nope'"); + }); + + it('ships the pilot skill selection by default', () => { + expect(SWEEP_SKILLS).toEqual([ + 'kernelcad', + 'kernelcad-authoring', + 'kernelcad-assemblies', + 'kernelcad-parts', + ]); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run eval/lib/systemPrompt.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// eval/lib/systemPrompt.ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** Pilot-parity skill selection for MUSE/benchmark sweeps. */ +export const SWEEP_SKILLS = [ + 'kernelcad', + 'kernelcad-authoring', + 'kernelcad-assemblies', + 'kernelcad-parts', +] as const; + +export const SKILLS_ROOT = resolve('src/agent/skills'); + +/** Concatenate the SKILL.md files for the given skill dirs (sorted, safe). */ +export function buildSystemPrompt( + skillDirs: readonly string[], + root: string = SKILLS_ROOT, +): string { + const parts: string[] = []; + for (const name of [...skillDirs].sort()) { + const path = join(root, name, 'SKILL.md'); + if (!existsSync(path)) { + throw new Error(`SKILL.md not found for skill '${name}' at ${path}`); + } + parts.push(readFileSync(path, 'utf8')); + } + return parts.join('\n\n---\n\n'); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run eval/lib/systemPrompt.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add eval/lib/systemPrompt.ts eval/lib/systemPrompt.test.ts +git commit -m "eval: add selectable system-prompt builder for sweep runs" +``` + +--- + +### Task 3: Bounded-concurrency pool + +**Files:** +- Create: `eval/lib/pool.ts` +- Test: `eval/lib/pool.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// eval/lib/pool.test.ts +import { describe, expect, it } from 'vitest'; +import { mapPool } from './pool'; + +describe('mapPool', () => { + it('preserves result order', async () => { + const out = await mapPool([3, 1, 2], 2, async (n) => n * 10); + expect(out).toEqual([30, 10, 20]); + }); + + it('never exceeds the concurrency limit', async () => { + let active = 0; + let peak = 0; + await mapPool(Array.from({ length: 10 }, (_, i) => i), 3, async () => { + active++; + peak = Math.max(peak, active); + await new Promise((r) => setTimeout(r, 5)); + active--; + return true; + }); + expect(peak).toBeLessThanOrEqual(3); + expect(peak).toBeGreaterThan(1); + }); + + it('rejects with the first worker error', async () => { + await expect( + mapPool([1, 2], 2, async (n) => { + if (n === 1) throw new Error('boom'); + return n; + }), + ).rejects.toThrow('boom'); + }); + + it('handles an empty input', async () => { + expect(await mapPool([], 4, async () => 1)).toEqual([]); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run eval/lib/pool.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// eval/lib/pool.ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors + +/** + * Bounded-concurrency map. Results keep input order. Rejects with the first + * worker rejection; caller is responsible for catching per-item errors when + * item isolation is required. + */ +export async function mapPool( + items: readonly T[], + limit: number, + worker: (item: T, index: number) => Promise, +): Promise { + if (limit < 1) throw new Error(`mapPool: limit must be >= 1, got ${limit}`); + const results = new Array(items.length); + let next = 0; + const runners = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await worker(items[index], index); + } + }); + await Promise.all(runners); + return results; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run eval/lib/pool.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add eval/lib/pool.ts eval/lib/pool.test.ts +git commit -m "eval: add bounded-concurrency map helper" +``` + +--- + +### Task 4: Per-case state machine + +**Files:** +- Create: `eval/lib/museState.ts` +- Test: `eval/lib/museState.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// eval/lib/museState.test.ts +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { isAtLeast, readState, writeState, type CaseState } from './museState'; + +const BASE: Omit = { + phase: 'generated', + attempts: 2, + tokens: { in: 100, out: 20 }, + protocol: 'muse-v1', +}; + +describe('museState', () => { + it('writes and reads a state atomically', () => { + const dir = mkdtempSync(join(tmpdir(), 'mstate-')); + writeState(dir, { ...BASE, updatedAt: '2026-09-19T00:00:00Z' }); + const out = readState(dir); + expect(out).toEqual({ ...BASE, updatedAt: '2026-09-19T00:00:00Z' }); + }); + + it('returns null when no state file exists', () => { + const dir = mkdtempSync(join(tmpdir(), 'mstate-')); + expect(readState(dir)).toBeNull(); + }); + + it('orders phases and treats infra_error as never at-target', () => { + expect(isAtLeast('judged', 'scored')).toBe(true); + expect(isAtLeast('generated', 'scored')).toBe(false); + expect(isAtLeast('pending', 'generated')).toBe(false); + expect(isAtLeast('infra_error', 'pending')).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run eval/lib/museState.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// eval/lib/museState.ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export type MusePhase = 'pending' | 'generated' | 'scored' | 'judged' | 'infra_error'; + +export interface CaseState { + phase: MusePhase; + attempts: number; + tokens: { in: number; out: number }; + firstFailureCode?: string; + /** Generation wall-clock, preserved so resumed scoring can report totals. */ + generationMs?: number; + error?: string; + protocol: string; + updatedAt: string; +} + +const PHASE_ORDER: Record, number> = { + pending: 0, + generated: 1, + scored: 2, + judged: 3, +}; + +export function statePath(caseDir: string): string { + return join(caseDir, 'state.json'); +} + +export function writeState(caseDir: string, state: CaseState): void { + const target = statePath(caseDir); + const tmp = `${target}.tmp`; + writeFileSync(tmp, JSON.stringify(state, null, 2)); + renameSync(tmp, target); +} + +export function readState(caseDir: string): CaseState | null { + const path = statePath(caseDir); + if (!existsSync(path)) return null; + return JSON.parse(readFileSync(path, 'utf8')) as CaseState; +} + +/** True when `phase` is at or past `target`. infra_error is never at-target. */ +export function isAtLeast(phase: MusePhase, target: MusePhase): boolean { + if (phase === 'infra_error' || target === 'infra_error') return false; + return PHASE_ORDER[phase] >= PHASE_ORDER[target]; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run eval/lib/museState.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add eval/lib/museState.ts eval/lib/museState.test.ts +git commit -m "eval: add per-case sweep state machine" +``` + +--- + +### Task 5: MUSE aggregation arithmetic + +**Files:** +- Create: `eval/lib/museAggregate.ts` +- Test: `eval/lib/museAggregate.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// eval/lib/museAggregate.test.ts +import { describe, expect, it } from 'vitest'; +import { aggregateMuseSamples, type MuseSample } from './museAggregate'; + +const judged = (overrides: Partial = {}): MuseSample => ({ + case: 'c', + sandboxOk: true, + overlapFree: true, + categories: { + assembly_readiness: 1, + joint_design: 0, + tolerance: 1, + functional_adaptation: 1, + usage_stability: 0, + manufacturability: 1, + }, + ...overrides, +}); + +describe('aggregateMuseSamples', () => { + it('maps the six categories to the three pillars and final', () => { + const { row } = aggregateMuseSamples([judged()], { model: 'm+kcad' }); + expect(row.functional).toBe(100); + expect(row.robust).toBe(0); + expect(row.functionality).toBe(50); + expect(row.well_toleranced).toBe(100); + expect(row.manufacturable).toBe(100); + expect(row.manufacturability).toBe(100); + expect(row.assembly_ready).toBe(100); + expect(row.connectable).toBe(0); + expect(row.assemblability).toBe(50); + expect(row.final).toBe(66.67); + expect(row.sandbox).toBe(100); + expect(row.overlap_free).toBe(100); + expect(row.judged).toBe(1); + expect(row.cases).toBe(1); + }); + + it('zeroes all categories when stage 1 or overlap fails', () => { + const { row } = aggregateMuseSamples( + [judged({ sandboxOk: false }), judged({ overlapFree: false, categories: judged().categories })], + { model: 'm+kcad' }, + ); + expect(row.final).toBe(0); + expect(row.robust).toBe(0); + expect(row.well_toleranced).toBe(0); + expect(row.sandbox).toBe(0); + }); + + it('averages across cases and counts judged samples', () => { + const { row } = aggregateMuseSamples( + [judged(), judged({ sandboxOk: false })], + { model: 'm+kcad' }, + ); + expect(row.cases).toBe(2); + expect(row.judged).toBe(1); + expect(row.sandbox).toBe(50); + expect(row.final).toBe(33.33); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run eval/lib/museAggregate.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// eval/lib/museAggregate.ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// +// Aggregation mirrors MUSE's leaderboard arithmetic +// (`scripts/bench_evaluate/generate_latex_tables_gemini.py` at commit 547a724^): +// functionality = mean(functional_adaptation, usage_stability) +// manufacturability = mean(tolerance, manufacturability) +// assemblability = mean(assembly_readiness, joint_design) +// final = mean(functionality, manufacturability, assemblability) +// Any stage-1/stage-2 failure zeroes all six categories. + +export interface JudgeCategories { + assembly_readiness: number; + joint_design: number; + tolerance: number; + functional_adaptation: number; + usage_stability: number; + manufacturability: number; +} + +export interface MuseSample { + case: string; + sandboxOk: boolean; + overlapFree: boolean; + /** Judge categories (0/1); absent when the judge was not run. */ + categories?: JudgeCategories; +} + +export interface MuseLeaderboardRow { + model: string; + judged: number; + cases: number; + sandbox: number; + overlap_free: number; + functionality: number; + manufacturability: number; + assemblability: number; + final: number; + functional: number; + robust: number; + well_toleranced: number; + manufacturable: number; + assembly_ready: number; + connectable: number; +} + +export const ZERO_CATEGORIES: JudgeCategories = { + assembly_readiness: 0, + joint_design: 0, + tolerance: 0, + functional_adaptation: 0, + usage_stability: 0, + manufacturability: 0, +}; + +const pct = (x: number): number => Math.round(x * 10000) / 100; + +/** Effective categories for one sample after the funnel forced-zero rule. */ +export function effectiveCategories(sample: MuseSample): JudgeCategories | null { + if (!sample.sandboxOk || !sample.overlapFree) return null; + return sample.categories ?? null; +} + +function samplePillars(categories: JudgeCategories): { + functional: number; + robust: number; + well_toleranced: number; + manufacturable: number; + assembly_ready: number; + connectable: number; + functionality: number; + manufacturability: number; + assemblability: number; + final: number; +} { + const functionality = + (categories.functional_adaptation + categories.usage_stability) / 2; + const manufacturability = (categories.tolerance + categories.manufacturability) / 2; + const assemblability = (categories.assembly_readiness + categories.joint_design) / 2; + return { + functional: categories.functional_adaptation, + robust: categories.usage_stability, + well_toleranced: categories.tolerance, + manufacturable: categories.manufacturability, + assembly_ready: categories.assembly_readiness, + connectable: categories.joint_design, + functionality, + manufacturability, + assemblability, + final: (functionality + manufacturability + assemblability) / 3, + }; +} + +export function aggregateMuseSamples( + samples: readonly MuseSample[], + meta: { model: string }, +): { row: MuseLeaderboardRow; forcedZeroCases: string[]; judgedCases: number } { + const cases = samples.length || 1; + const forcedZeroCases: string[] = []; + let sandboxPass = 0; + let overlapPass = 0; + let judged = 0; + const sums = { + functionality: 0, + manufacturability: 0, + assemblability: 0, + final: 0, + functional: 0, + robust: 0, + well_toleranced: 0, + manufacturable: 0, + assembly_ready: 0, + connectable: 0, + }; + + for (const sample of samples) { + if (sample.sandboxOk) sandboxPass++; + if (sample.sandboxOk && sample.overlapFree) overlapPass++; + const effective = effectiveCategories(sample); + if (!effective) { + if (sample.categories) forcedZeroCases.push(sample.case); + continue; + } + judged++; + const pillars = samplePillars(effective); + sums.functional += pillars.functional; + sums.robust += pillars.robust; + sums.well_toleranced += pillars.well_toleranced; + sums.manufacturable += pillars.manufacturable; + sums.assembly_ready += pillars.assembly_ready; + sums.connectable += pillars.connectable; + sums.functionality += pillars.functionality; + sums.manufacturability += pillars.manufacturability; + sums.assemblability += pillars.assemblability; + sums.final += pillars.final; + } + + const row: MuseLeaderboardRow = { + model: meta.model, + judged, + cases: samples.length, + sandbox: pct(sandboxPass / cases), + overlap_free: pct(overlapPass / cases), + functionality: pct(sums.functionality / cases), + manufacturability: pct(sums.manufacturability / cases), + assemblability: pct(sums.assemblability / cases), + final: pct(sums.final / cases), + functional: pct(sums.functional / cases), + robust: pct(sums.robust / cases), + well_toleranced: pct(sums.well_toleranced / cases), + manufacturable: pct(sums.manufacturable / cases), + assembly_ready: pct(sums.assembly_ready / cases), + connectable: pct(sums.connectable / cases), + }; + return { row, forcedZeroCases, judgedCases: judged }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npx vitest run eval/lib/museAggregate.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add eval/lib/museAggregate.ts eval/lib/museAggregate.test.ts +git commit -m "eval: add MUSE pillar aggregation with forced-zero semantics" +``` + +--- + +### Task 6: Split `runTask` into `generateCase` + `scoreCase` + +**Files:** +- Modify: `eval/runner.ts` +- Test: `eval/runner.split.test.ts` (new), existing `eval/runner*.test.ts` must stay green + +- [ ] **Step 1: Run the existing runner tests to establish a green baseline** + +Run: `npx vitest run eval/runner.test.ts eval/runner.loop.test.ts eval/runner.bestOfN.test.ts` +Expected: PASS. + +- [ ] **Step 2: Write the failing split test** + +```ts +// eval/runner.split.test.ts +import { existsSync, mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { generateCase, scoreCase } from './runner'; +import { MockAgentClient } from './agent'; +import type { AgentResponse } from './types'; + +const TASK_DIR = join(__dirname, 'tasks', 'bracket-holes'); +const SKILLS = '# skills\n\nBe precise.'; +const hasCli = + existsSync(join(process.cwd(), 'dist/cli/index.js')) || Boolean(process.env.KERNELCAD_BIN); + +const GOOD_SCRIPT = [ + '```ts', + "export const bracket = box({ length: 40, width: 20, height: 5 });", + '```', +].join('\n'); + +describe.skipIf(!hasCli)('generateCase / scoreCase split', () => { + it('generates without scoring, then scores the written script', async () => { + const runDir = mkdtempSync(join(tmpdir(), 'split-')); + const agent = new MockAgentClient([ + { text: GOOD_SCRIPT, tokens_in: 10, tokens_out: 5 } satisfies AgentResponse, + ]); + const gen = await generateCase({ + taskDir: TASK_DIR, + runDir, + agent, + model: 'mock-model', + skillMd: SKILLS, + startedAt: '2026-09-19T00-00-00', + candidates: 1, + maxAttempts: 3, + maxTokens: 8000, + temperature: 0.2, + }); + const script = readFileSync(gen.outputScriptPath, 'utf8'); + expect(script.length).toBeGreaterThan(0); + expect(gen.tokensIn).toBe(10); + + const result = await scoreCase({ + taskDir: TASK_DIR, + runDir, + outputScriptPath: gen.outputScriptPath, + events: gen.events, + attempts: gen.attempts, + tokensIn: gen.tokensIn, + tokensOut: gen.tokensOut, + generationMs: gen.timeMs, + startedAt: '2026-09-19T00-00-00', + model: 'mock-model', + firstFailureCode: gen.firstFailureCode, + noScript: gen.status === 'no_script', + }); + expect(result.task).toBe('bracket-holes'); + expect(result.score).not.toBeNull(); + expect(readFileSync(join(runDir, 'score.json'), 'utf8')).toContain('"attempts"'); + }); +}); +``` + +- [ ] **Step 3: Run the test to verify it fails** + +Run: `npx vitest run eval/runner.split.test.ts` +Expected: FAIL — `generateCase`/`scoreCase` not exported (or the skip guard reports the CLI missing — set `KERNELCAD_BIN` or run `npm run build:cli` first). + +- [ ] **Step 4: Refactor `eval/runner.ts`** + +Replace the body of `runTask` with the split below. Full replacement for the exported functions (keep imports, constants, `variantTemperature`, `reduceHarnessScore` unchanged): + +```ts +export interface GenerateCaseArgs { + taskDir: string; + runDir: string; + agent: AgentClient; + model: string; + skillMd: string; + startedAt: string; + cookbook?: CookbookInjection; + candidates?: number; + maxAttempts?: number; + maxTokens?: number; + /** Sent on every call when candidates <= 1 (sweep protocol temperature). */ + temperature?: number; +} + +export interface GenerateCaseResult { + events: TranscriptEvent[]; + status: 'passed' | 'gate_failed' | 'no_script'; + attempts: number; + tokensIn: number; + tokensOut: number; + timeMs: number; + firstFailureCode?: string; + outputScriptPath: string; +} + +export async function generateCase(args: GenerateCaseArgs): Promise { + const taskDirAbs = resolve(args.taskDir); + const prompt = readFileSync(join(taskDirAbs, 'prompt.md'), 'utf8'); + + mkdirSync(args.runDir, { recursive: true }); + const outputScriptPath = join(args.runDir, 'output.kcad.ts'); + + const events: TranscriptEvent[] = []; + events.push({ kind: 'system_prompt', chars: args.skillMd.length }); + events.push({ kind: 'user_prompt', content: prompt }); + if (args.cookbook) { + events.push({ kind: 'cookbook_inject', query: args.cookbook.query, hits: args.cookbook.hits }); + } + + let attemptNo = 0; + let totalIn = 0; + let totalOut = 0; + let firstFailureCode: string | undefined; + const start = Date.now(); + + const candidates = args.candidates ?? 1; + const maxTokens = args.maxTokens ?? MAX_TOKENS; + + const loopResult = await runClosedLoop({ + prompt, + gateRunner: createWebGateRunner(), + extractScript, + buildRepairPrompt, + maxAttempts: args.maxAttempts ?? MAX_ATTEMPTS, + candidates, + scoreCandidate: async (scriptPath, report) => { + if (!report.ok) return null; + try { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) return null; + const harnessModule = await import(join(taskDirAbs, 'harness.ts')); + const hr = await harnessModule.default(scriptPath); + return reduceHarnessScore(hr); + } catch { + return null; + } + }, + writeScript: async (code: string) => { + writeFileSync(outputScriptPath, code); + return outputScriptPath; + }, + generate: async (messages: LoopMessage[], opts?: { variant?: number }) => { + attemptNo += 1; + const turnStart = Date.now(); + const temperature = + candidates > 1 ? variantTemperature(opts?.variant) : args.temperature; + const resp = await args.agent.generate({ + system: args.skillMd, + systemAddendum: args.cookbook?.systemPromptAddendum, + messages: messages.map((m) => ({ role: m.role, content: m.content })), + model: args.model, + max_tokens: maxTokens, + temperature, + }); + totalIn += resp.tokens_in; + totalOut += resp.tokens_out; + events.push({ + kind: 'turn', + attempt: attemptNo, + assistant_text: resp.text, + script_extracted: extractScript(resp.text), + tokens_in: resp.tokens_in, + tokens_out: resp.tokens_out, + ms: Date.now() - turnStart, + }); + return { text: resp.text, tokensIn: resp.tokens_in, tokensOut: resp.tokens_out }; + }, + onEvent: (e) => { + if (e.type === 'gate_report') { + const failing = e.report.verdicts.filter((v) => !v.ok); + events.push({ + kind: 'evaluate', + attempt: attemptNo, + ok: e.report.ok, + diagnostics: failing.map((v) => ({ + code: v.code ?? v.gate, + message: v.message, + hint: v.hint, + featureId: v.locus, + })), + }); + if (firstFailureCode === undefined && failing.length > 0) { + firstFailureCode = failing[0].code ?? failing[0].gate; + } + } + }, + }); + + if (loopResult.status === 'no_script') { + writeFileSync(outputScriptPath, '// (no script extracted from any attempt)'); + } + + return { + events, + status: loopResult.status, + attempts: loopResult.attempts, + tokensIn: totalIn, + tokensOut: totalOut, + timeMs: Date.now() - start, + firstFailureCode, + outputScriptPath, + }; +} + +export interface ScoreCaseArgs { + taskDir: string; + runDir: string; + outputScriptPath: string; + events?: TranscriptEvent[]; + attempts: number; + tokensIn: number; + tokensOut: number; + generationMs: number; + startedAt: string; + model: string; + firstFailureCode?: string; + /** True when generation never extracted a script — skips the evaluate call. */ + noScript?: boolean; +} + +export async function scoreCase(args: ScoreCaseArgs): Promise { + const taskDirAbs = resolve(args.taskDir); + const taskName = taskDirAbs.split('/').pop() ?? 'unknown'; + const events = args.events ?? []; + const scoringStart = Date.now(); + + const finalEvaluate: EvaluateResult = args.noScript + ? { + ok: false, + diagnostics: [ + { code: 'eval.no-script-extracted', message: 'No script extracted from any attempt.' }, + ], + } + : await evaluateScript(args.outputScriptPath); + let firstFailureCode = args.firstFailureCode; + if (firstFailureCode === undefined && !finalEvaluate.ok && finalEvaluate.diagnostics.length > 0) { + firstFailureCode = finalEvaluate.diagnostics[0].code; + } + + let harnessResult: HarnessResult; + if (finalEvaluate.ok) { + const harnessModule = await import(join(taskDirAbs, 'harness.ts')); + harnessResult = await harnessModule.default(args.outputScriptPath, { + taskDir: taskDirAbs, + runDir: args.runDir, + }); + } else { + harnessResult = { gates: { 'evaluates clean': false }, scored: {} }; + } + + events.push({ + kind: 'score', + gates: harnessResult.gates, + scored: harnessResult.scored, + }); + + const score = computeScore(harnessResult, { + attempts: args.attempts, + tokens_in: args.tokensIn, + tokens_out: args.tokensOut, + time_ms: args.generationMs + (Date.now() - scoringStart), + firstFailureCode, + }); + + writeFileSync(join(args.runDir, 'score.json'), JSON.stringify(score, null, 2)); + writeFileSync( + join(args.runDir, 'transcript.md'), + renderTranscript({ + task: taskName, + model: args.model, + started_at: args.startedAt, + events, + score, + }), + ); + + return { task: taskName, score }; +} + +export async function runTask(args: RunTaskArgs): Promise { + const gen = await generateCase({ + taskDir: args.taskDir, + runDir: args.runDir, + agent: args.agent, + model: args.model, + skillMd: args.skillMd, + startedAt: args.startedAt, + cookbook: args.cookbook, + candidates: args.candidates, + maxAttempts: args.maxAttempts, + maxTokens: args.maxTokens, + temperature: args.temperature, + }); + return scoreCase({ + taskDir: args.taskDir, + runDir: args.runDir, + outputScriptPath: gen.outputScriptPath, + events: gen.events, + attempts: gen.attempts, + tokensIn: gen.tokensIn, + tokensOut: gen.tokensOut, + generationMs: gen.timeMs, + startedAt: args.startedAt, + model: args.model, + firstFailureCode: gen.firstFailureCode, + noScript: gen.status === 'no_script', + }); +} +``` + +Also extend `RunTaskArgs` with `maxAttempts?`, `maxTokens?`, `temperature?` (same optional semantics). Add `EvaluateResult` to the type imports at the top of `eval/runner.ts` (`import type { AgentClient, TranscriptEvent, TaskResult, HarnessResult, EvaluateResult } from './types';`). + +Note: the no-script placeholder path now flows through `scoreCase` with `noScript: true`; behavior is identical to the old code (old code skipped `evaluateScript` entirely for that path). + +- [ ] **Step 5: Run all runner tests** + +Run: `npx vitest run eval/runner.test.ts eval/runner.loop.test.ts eval/runner.bestOfN.test.ts eval/runner.split.test.ts` +Expected: PASS (split test runs when the CLI is built). + +- [ ] **Step 6: Commit** + +```bash +git add eval/runner.ts eval/runner.split.test.ts +git commit -m "eval: split runTask into generateCase and scoreCase for sweep resume" +``` + +--- + +### Task 7: MUSE judge wrapper + `judgeCase` + +**Files:** +- Create: `eval/oracle/museJudgeWrapper.py` +- Create: `scripts/museJudge.ts` +- Test: `scripts/museJudge.test.ts` (arg construction + forced-zero; no network) + +- [ ] **Step 1: Write the failing test** + +```ts +// scripts/museJudge.test.ts +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { judgeCase, mapJudgePayload } from './museJudge'; + +describe('mapJudgePayload', () => { + it('maps six categories and computes overall fallback', () => { + const out = mapJudgePayload({ + overall_score_normalized: 0.5, + items: [ + { category_en: 'Assembly Readiness', score: 1 }, + { category_en: 'Joint Design', score: 0 }, + { category_en: 'Tolerance', score: 1 }, + { category_en: 'Functional Adaptation', score: 1 }, + { category_en: 'Usage Stability', score: 0 }, + { category_en: 'Manufacturability', score: 1 }, + ], + }); + expect(out.categories.assembly_readiness).toBe(1); + expect(out.categories.joint_design).toBe(0); + expect(out.overall).toBe(0.5); + }); + + it('writes a forced-zero judge.json without spawning the wrapper', async () => { + const dir = mkdtempSync(join(tmpdir(), 'judge-')); + const spawn = vi.fn(); + const result = await judgeCase( + { + caseName: 'stool', + datasetCaseDir: '/nonexistent', + candidatePng: '/nonexistent/render.png', + outPath: join(dir, 'judge.json'), + museRoot: '/nonexistent', + pythonBin: 'python3', + baseUrl: 'https://example.invalid', + model: 'judge', + }, + false, + spawn, + ); + expect(result.forcedZero).toBe(true); + expect(result.overall).toBe(0); + expect(spawn).not.toHaveBeenCalled(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run scripts/museJudge.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement the Python wrapper** + +```python +#!/usr/bin/env python3 +"""Score one kernelCAD sample with MUSE's own alignment judge. + +Calls MUSE's published functions directly (`_run_alignment_judge`, +`_load_score_system_prompt`, `_build_alignment_prompt`) with MUSE's judge +model and temperature. Writes the parsed result to --out as JSON. + +Exit code is always 0; transport errors are reported as {"error": ...}. +""" +import argparse +import json +import os +import sys +from pathlib import Path + +CATEGORY_KEYS = { + "assembly readiness": "assembly_readiness", + "joint design": "joint_design", + "tolerance": "tolerance", + "functional adaptation": "functional_adaptation", + "usage stability": "usage_stability", + "manufacturability": "manufacturability", +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--muse-root", required=True) + parser.add_argument("--case-name", required=True) + parser.add_argument("--case-dir", required=True) + parser.add_argument("--candidate-png", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--model", default="google/gemini-3.1-pro") + parser.add_argument("--base-url", default="https://api.deepinfra.com/v1/openai") + parser.add_argument("--api-key-env", default="DEEPINFRA_API_KEY") + parser.add_argument("--timeout", type=int, default=180) + args = parser.parse_args() + + api_key = os.environ.get(args.api_key_env) + if not api_key: + print(json.dumps({"error": f"missing env {args.api_key_env}"})) + return 0 + + muse_root = Path(args.muse_root).resolve() + sys.path.insert(0, str(muse_root / "src")) + try: + from judge_system.reverse_pipeline import ( # type: ignore + _load_score_system_prompt, + _run_alignment_judge, + ) + except Exception as exc: # pragma: no cover - env misconfiguration + print(json.dumps({"error": f"cannot import MUSE judge from {muse_root}: {exc}"})) + return 0 + + case_dir = Path(args.case_dir).resolve() + render_only = set() + render_list = muse_root / "src" / "judge_system" / "render_only_cases.txt" + if render_list.exists(): + for line in render_list.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + render_only.add(line) + if args.case_name in render_only: + reference = case_dir / f"{args.case_name}_stp_render.png" + else: + reference = case_dir / f"{args.case_name}.png" + + candidate = Path(args.candidate_png).resolve() + if not candidate.exists(): + print(json.dumps({"error": f"candidate render missing: {candidate}"})) + return 0 + if not reference.exists(): + print(json.dumps({"error": f"reference image missing: {reference}"})) + return 0 + + try: + payload = _run_alignment_judge( + api_key=api_key, + base_url=args.base_url, + model=args.model, + timeout_seconds=args.timeout, + system_prompt=_load_score_system_prompt(), + task_text=(case_dir / "design_description.md").read_text(encoding="utf-8"), + rubric_text=(case_dir / "evaluation_rubric.md").read_text(encoding="utf-8"), + candidate_svg_png=candidate, + reference_png=reference, + ) + except Exception as exc: + print(json.dumps({"error": f"judge call failed: {exc}"})) + return 0 + + categories = {} + for item in payload.get("items", []): + key = CATEGORY_KEYS.get(str(item.get("category_en", "")).strip().lower()) + if key: + try: + categories[key] = 1.0 if float(item.get("score", 0) or 0) >= 0.5 else 0.0 + except (TypeError, ValueError): + categories[key] = 0.0 + + overall = payload.get("overall_score_normalized") + if overall is None: + overall = payload.get("overall_score") + try: + overall_value = float(overall or 0.0) + if overall_value > 1: + overall_value = overall_value / 100.0 + except (TypeError, ValueError): + overall_value = 0.0 + + out = { + "overall": overall_value, + "categories": categories, + "summary": str(payload.get("overall_summary", "") or ""), + "items": payload.get("items", []), + "judge_model": args.model, + "judge_base_url": args.base_url, + "candidate_png": str(candidate), + "reference_png": str(reference), + } + Path(args.out).write_text(json.dumps(out, indent=2), encoding="utf-8") + print(json.dumps({"ok": True, "out": args.out})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 4: Implement `scripts/museJudge.ts`** + +```ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { spawn as nodeSpawn } from 'node:child_process'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + ZERO_CATEGORIES, + type JudgeCategories, +} from '../eval/lib/museAggregate'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const WRAPPER_PY = resolve(__dirname, '../eval/oracle/museJudgeWrapper.py'); + +export interface JudgeCaseArgs { + caseName: string; + datasetCaseDir: string; + candidatePng: string; + outPath: string; + museRoot: string; + pythonBin: string; + baseUrl: string; + model: string; + timeoutMs?: number; +} + +export interface JudgeResult { + overall: number; + categories: JudgeCategories; + forcedZero: boolean; + forcedZeroReason?: string; + summary?: string; +} + +type SpawnFn = typeof nodeSpawn; + +const CATEGORY_KEYS: Record = { + 'assembly readiness': 'assembly_readiness', + 'joint design': 'joint_design', + tolerance: 'tolerance', + 'functional adaptation': 'functional_adaptation', + 'usage stability': 'usage_stability', + manufacturability: 'manufacturability', +}; + +export function mapJudgePayload(payload: Record): { + overall: number; + categories: JudgeCategories; + summary: string; +} { + const categories: JudgeCategories = { ...ZERO_CATEGORIES }; + const items = Array.isArray(payload.items) ? payload.items : []; + for (const raw of items) { + const item = raw as { category_en?: unknown; score?: unknown }; + const key = CATEGORY_KEYS[String(item.category_en ?? '').trim().toLowerCase()]; + if (!key) continue; + const score = Number(item.score ?? 0); + categories[key] = Number.isFinite(score) && score >= 0.5 ? 1 : 0; + } + let overall = Number(payload.overall ?? payload.overall_score_normalized ?? 0); + if (!Number.isFinite(overall) || overall <= 0) { + const values = Object.values(categories); + overall = values.reduce((a, b) => a + b, 0) / 6; + } + return { overall, categories, summary: String(payload.summary ?? '') }; +} + +function runWrapper( + args: JudgeCaseArgs, + spawnFn: SpawnFn, +): Promise> { + return new Promise((resolvePromise, reject) => { + const child = spawnFn(args.pythonBin, [ + WRAPPER_PY, + '--muse-root', args.museRoot, + '--case-name', args.caseName, + '--case-dir', args.datasetCaseDir, + '--candidate-png', args.candidatePng, + '--out', args.outPath, + '--model', args.model, + '--base-url', args.baseUrl, + ], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d.toString())); + child.stderr.on('data', (d) => (stderr += d.toString())); + child.on('error', reject); + child.on('close', (code) => { + const line = stdout.split('\n').map((l) => l.trim()).reverse().find((l) => l.startsWith('{')); + if (!line) { + reject(new Error(`judge wrapper exited ${code}: ${stderr.slice(0, 300)}`)); + return; + } + const parsed = JSON.parse(line) as Record; + if (typeof parsed.error === 'string') { + reject(new Error(parsed.error)); + return; + } + resolvePromise(parsed); + }); + }); +} + +export async function judgeCase( + args: JudgeCaseArgs, + stage12Ok: boolean, + spawnFn: SpawnFn = nodeSpawn, +): Promise { + mkdirSync(dirname(args.outPath), { recursive: true }); + if (!stage12Ok) { + const result: JudgeResult = { + overall: 0, + categories: { ...ZERO_CATEGORIES }, + forcedZero: true, + forcedZeroReason: 'stage 1 sandbox or stage 2 overlap failed; MUSE forces all categories to 0', + }; + writeFileSync(args.outPath, JSON.stringify(result, null, 2)); + return result; + } + const payload = await runWrapper(args, spawnFn); + const mapped = mapJudgePayload(payload); + const result: JudgeResult = { + overall: mapped.overall, + categories: mapped.categories, + forcedZero: false, + summary: mapped.summary, + }; + writeFileSync(args.outPath, JSON.stringify({ ...result, raw: payload }, null, 2)); + return result; +} +``` + +- [ ] **Step 5: Run the test** + +Run: `npx vitest run scripts/museJudge.test.ts` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add eval/oracle/museJudgeWrapper.py scripts/museJudge.ts scripts/museJudge.test.ts +git commit -m "eval: add MUSE alignment-judge wrapper and TS judge module" +``` + +--- + +### Task 8: Preflight + +**Files:** +- Create: `scripts/musePreflight.ts` +- Test: `scripts/musePreflight.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// scripts/musePreflight.test.ts +import { describe, expect, it } from 'vitest'; +import { runPreflight, formatPreflight, type PreflightDeps } from './musePreflight'; + +function deps(overrides: Partial = {}): PreflightDeps { + return { + env: { + DEEPINFRA_API_KEY: 'x', + MUSE_ROOT: '/muse', + MUSE_PYTHON: '/muse/.venv/bin/python', + KERNELCAD_BIN: './dist/cli/index.js', + }, + exists: () => true, + run: async () => ({ code: 0, stdout: 'ok\n', stderr: '' }), + fetchImpl: (async () => + new Response(JSON.stringify({ data: [{ id: 'google/gemini-3.1-pro' }] }), { + status: 200, + })) as unknown as typeof fetch, + caseCount: () => 106, + ...overrides, + }; +} + +describe('runPreflight', () => { + it('passes when every check is green', async () => { + const report = await runPreflight(deps()); + expect(report.ok).toBe(true); + expect(report.checks.every((c) => c.ok)).toBe(true); + }); + + it('fails when the API key is missing', async () => { + const report = await runPreflight( + deps({ env: { MUSE_ROOT: '/muse', MUSE_PYTHON: '/muse/.venv/bin/python' } }), + ); + expect(report.ok).toBe(false); + expect(report.checks.find((c) => c.name === 'deepinfra key')?.ok).toBe(false); + }); + + it('fails when the judge model is absent from the endpoint', async () => { + const report = await runPreflight( + deps({ + fetchImpl: (async () => + new Response(JSON.stringify({ data: [{ id: 'other' }] }), { + status: 200, + })) as unknown as typeof fetch, + }), + ); + expect(report.checks.find((c) => c.name === 'judge model')?.ok).toBe(false); + }); + + it('formats one line per check', () => { + const text = formatPreflight({ + ok: false, + checks: [ + { name: 'a', ok: true, detail: 'fine' }, + { name: 'b', ok: false, detail: 'broken' }, + ], + }); + expect(text).toContain('PASS a: fine'); + expect(text).toContain('FAIL b: broken'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run scripts/musePreflight.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { execFile } from 'node:child_process'; +import { existsSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +export interface PreflightCheck { + name: string; + ok: boolean; + detail: string; +} + +export interface PreflightReport { + ok: boolean; + checks: PreflightCheck[]; +} + +export interface PreflightDeps { + env: Record; + exists: (path: string) => boolean; + run: (cmd: string, args: string[]) => Promise<{ code: number; stdout: string; stderr: string }>; + fetchImpl: typeof fetch; + caseCount: () => number; +} + +const JUDGE_MODEL = 'google/gemini-3.1-pro'; +const DEFAULT_BASE_URL = 'https://api.deepinfra.com/v1/openai'; + +function defaultDeps(): PreflightDeps { + const museRoot = process.env.MUSE_ROOT ?? join(process.env.HOME ?? '', 'projects/muse'); + return { + env: process.env as Record, + exists: existsSync, + run: (cmd, args) => + new Promise((res) => { + execFile(cmd, args, { timeout: 120_000 }, (err, stdout, stderr) => { + res({ + code: err ? ((err as { code?: number }).code ?? 1) : 0, + stdout: String(stdout ?? ''), + stderr: String(stderr ?? ''), + }); + }); + }), + fetchImpl: fetch, + caseCount: () => { + const dir = join(museRoot, 'data/muse/cases'); + return existsSync(dir) ? readdirSync(dir).length : 0; + }, + }; +} + +export async function runPreflight(deps: PreflightDeps): Promise { + const checks: PreflightCheck[] = []; + const push = (name: string, ok: boolean, detail: string) => checks.push({ name, ok, detail }); + + const kcadBin = deps.env.KERNELCAD_BIN ?? './dist/cli/index.js'; + const kcadPath = kcadBin.endsWith('.js') ? resolve(kcadBin) : kcadBin; + push('kernelcad cli', deps.exists(kcadPath), kcadPath); + + const museRoot = deps.env.MUSE_ROOT ?? join(deps.env.HOME ?? '', 'projects/muse'); + push('muse checkout', deps.exists(resolve(museRoot, 'src/judge_system')), museRoot); + + const python = deps.env.MUSE_PYTHON ?? join(museRoot, '.venv/bin/python'); + const importCheck = await deps.run(python, [ + '-c', + `import sys; sys.path.insert(0, ${JSON.stringify(resolve(museRoot, 'src'))}); import cadquery, vtk, requests; from judge_system import reverse_pipeline; print('ok')`, + ]); + push('muse python imports', importCheck.code === 0 && importCheck.stdout.includes('ok'), importCheck.stderr.trim().slice(0, 200) || python); + + const key = deps.env.DEEPINFRA_API_KEY; + push('deepinfra key', Boolean(key && key.length > 0), key ? 'set' : 'DEEPINFRA_API_KEY missing'); + + const baseUrl = (deps.env.DEEPINFRA_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/+$/, ''); + try { + const resp = await deps.fetchImpl(`${baseUrl}/models`, { + headers: { Authorization: `Bearer ${key ?? ''}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array<{ id?: string }> }; + const ids = (data.data ?? []).map((m) => m.id ?? ''); + push('judge model', resp.ok && ids.includes(JUDGE_MODEL), `${JUDGE_MODEL} @ ${baseUrl}`); + } catch (err) { + push('judge model', false, err instanceof Error ? err.message : String(err)); + } + + const count = deps.caseCount(); + push('muse cases', count === 106, `${count} cases (expected 106)`); + + return { ok: checks.every((c) => c.ok), checks }; +} + +export function formatPreflight(report: PreflightReport): string { + return report.checks + .map((c) => `${c.ok ? 'PASS' : 'FAIL'} ${c.name}: ${c.detail}`) + .join('\n'); +} + +async function main(): Promise { + const report = await runPreflight(defaultDeps()); + console.log(formatPreflight(report)); + process.exit(report.ok ? 0 : 1); +} + +const isEntrypoint = (() => { + if (!process.argv[1]) return false; + try { + return import.meta.url === new URL(`file://${process.argv[1]}`).href; + } catch { + return false; + } +})(); + +if (isEntrypoint) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} +``` + +- [ ] **Step 4: Run the test** + +Run: `npx vitest run scripts/musePreflight.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/musePreflight.ts scripts/musePreflight.test.ts +git commit -m "eval: add MUSE sweep preflight checks" +``` + +--- + +### Task 9: Batch sweep CLI + +**Files:** +- Create: `scripts/runMuseSweep.ts` + +- [ ] **Step 1: Implement the CLI** + +```ts +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { execFile, execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { MockAgentClient } from '../eval/agent'; +import { OpenAICompatAgentClient } from '../eval/agentOpenAICompat'; +import { generateCase, scoreCase } from '../eval/runner'; +import { buildSystemPrompt, SWEEP_SKILLS } from '../eval/lib/systemPrompt'; +import { mapPool } from '../eval/lib/pool'; +import { isAtLeast, readState, writeState, type MusePhase } from '../eval/lib/museState'; +import { judgeCase } from './museJudge'; +import { runPreflight, formatPreflight, type PreflightReport } from './musePreflight'; +import { writeReports } from './museReport'; +import type { AgentClient, AgentResponse, TaskResult } from '../eval/types'; + +const TASKS_DIR = resolve('eval/tasks'); +const RUNS_DIR = resolve('eval/runs'); +const DEFAULT_MODEL = 'deepseek-ai/DeepSeek-V4.1-Flash'; +const DEFAULT_BASE_URL = 'https://api.deepinfra.com/v1/openai'; +const JUDGE_MODEL = 'google/gemini-3.1-pro'; +const PROTOCOL = 'muse-v1'; + +interface SweepConfig { + runId: string; + runRoot: string; + cases: string[]; + workers: number; + model: string; + baseUrl: string; + temperature: number; + maxAttempts: number; + maxTokens: number; + skills: string[]; + skipJudge: boolean; + force: Set; + maxTokensIn: number; + mockFixture?: string; + startedAt: string; + env: Record; +} + +export interface CaseOutcome { + case: string; + status: 'ok' | 'skipped' | 'stopped' | 'budget' | 'infra'; + phase?: MusePhase; + finalScore?: number; + attempts?: number; + timeMs?: number; + error?: string; +} + +function parseArgs(argv: string[]): SweepConfig { + const flagValue = (name: string): string | undefined => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; + }; + const has = (name: string): boolean => argv.includes(name); + const list = (name: string): string[] => { + const v = flagValue(name); + return v ? v.split(',').map((s) => s.trim()).filter(Boolean) : []; + }; + + const model = flagValue('--model') ?? DEFAULT_MODEL; + const slug = model.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, ''); + let sha = 'nogit'; + let dirty = ''; + try { + sha = execFileSync('git', ['rev-parse', '--short=7', 'HEAD'], { encoding: 'utf8' }).trim(); + dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0 ? '-dirty' : ''; + } catch { + // not a git checkout — leave as nogit + } + const ts = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-'); + const runId = flagValue('--run-id') ?? `muse106-${sha}${dirty}-${slug}-${ts}`; + + return { + runId, + runRoot: join(RUNS_DIR, runId), + cases: list('--cases'), + workers: Number(flagValue('--workers') ?? 6), + model, + baseUrl: flagValue('--base-url') ?? DEFAULT_BASE_URL, + temperature: Number(flagValue('--temperature') ?? 0.2), + maxAttempts: Number(flagValue('--max-attempts') ?? 3), + maxTokens: Number(flagValue('--max-tokens') ?? 8000), + skills: flagValue('--skills') ? list('--skills') : [...SWEEP_SKILLS], + skipJudge: has('--skip-judge'), + force: new Set(list('--force')), + maxTokensIn: Number(flagValue('--max-tokens-in') ?? 25_000_000), + mockFixture: flagValue('--mock-fixture'), + startedAt: new Date().toISOString().replace(/\..+$/, '').replace(/:/g, '-'), + env: process.env as Record, + }; +} + +function discoverCases(filter: string[]): string[] { + const all = readdirSync(TASKS_DIR) + .filter((name) => name.startsWith('muse-')) + .filter((name) => { + const dir = join(TASKS_DIR, name); + return ( + statSync(dir).isDirectory() && + existsSync(join(dir, 'prompt.md')) && + existsSync(join(dir, 'harness.ts')) + ); + }) + .map((name) => name.replace(/^muse-/, '')); + if (filter.length === 0) return all.sort(); + const wanted = new Set(filter); + return all.filter((c) => wanted.has(c)).sort(); +} + +function readScoreResult(caseDir: string): TaskResult | null { + const path = join(caseDir, 'score.json'); + if (!existsSync(path)) return null; + return { task: caseDir.split('/').pop() ?? 'unknown', score: JSON.parse(readFileSync(path, 'utf8')) }; +} + +/** The no-script placeholder written by generateCase. */ +function isNoScriptArtifact(outputScriptPath: string): boolean { + if (!existsSync(outputScriptPath)) return false; + return readFileSync(outputScriptPath, 'utf8').startsWith('// (no script extracted'); +} + +function readCachedAgentResponses(fixturePath: string): AgentResponse[] { + const data = JSON.parse(readFileSync(fixturePath, 'utf8')) as { responses: AgentResponse[] }; + return data.responses; +} + +async function runOneCase( + caseName: string, + cfg: SweepConfig, + agent: AgentClient, + skillMd: string, + totals: { tokensIn: number; tokensOut: number }, +): Promise { + const taskDir = join(TASKS_DIR, `muse-${caseName}`); + const caseDir = join(cfg.runRoot, 'cases', caseName); + mkdirSync(caseDir, { recursive: true }); + + if (existsSync(join(cfg.runRoot, 'STOP'))) { + return { case: caseName, status: 'stopped' }; + } + if (totals.tokensIn >= cfg.maxTokensIn) { + return { case: caseName, status: 'budget' }; + } + + const targetPhase: MusePhase = cfg.skipJudge ? 'scored' : 'judged'; + let state = readState(caseDir); + if ( + !cfg.force.has(caseName) && + state !== null && + isAtLeast(state.phase, targetPhase) + ) { + return { case: caseName, status: 'skipped', phase: state.phase }; + } + + try { + const outputScriptPath = join(caseDir, 'output.kcad.ts'); + let result: TaskResult; + let generationMs = state?.generationMs ?? 0; + + const canReuseGeneration = + !cfg.force.has(caseName) && + state !== null && + isAtLeast(state.phase, 'generated') && + existsSync(outputScriptPath); + + if (!canReuseGeneration) { + const gen = await generateCase({ + taskDir, + runDir: caseDir, + agent, + model: cfg.model, + skillMd, + startedAt: cfg.startedAt, + candidates: 1, + maxAttempts: cfg.maxAttempts, + maxTokens: cfg.maxTokens, + temperature: cfg.temperature, + }); + generationMs = gen.timeMs; + totals.tokensIn += gen.tokensIn; + totals.tokensOut += gen.tokensOut; + writeState(caseDir, { + phase: 'generated', + attempts: gen.attempts, + tokens: { in: gen.tokensIn, out: gen.tokensOut }, + firstFailureCode: gen.firstFailureCode, + generationMs: gen.timeMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + state = readState(caseDir); + } + + const cached = readScoreResult(caseDir); + if (cached !== null && !cfg.force.has(caseName) && isAtLeast(state!.phase, 'scored')) { + result = cached; + } else { + result = await scoreCase({ + taskDir, + runDir: caseDir, + outputScriptPath, + attempts: state?.attempts ?? 1, + tokensIn: state?.tokens.in ?? 0, + tokensOut: state?.tokens.out ?? 0, + generationMs, + startedAt: cfg.startedAt, + model: cfg.model, + firstFailureCode: state?.firstFailureCode, + noScript: isNoScriptArtifact(outputScriptPath), + }); + writeState(caseDir, { + phase: 'scored', + attempts: state?.attempts ?? 1, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: result.score?.firstFailureCode ?? state?.firstFailureCode, + generationMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + } + + if (!cfg.skipJudge) { + const metrics = result.score?.metrics ?? {}; + const stage12Ok = + metrics.muse_sandbox_ok === true && metrics.muse_overlap_free === true; + const candidatePng = String(metrics.muse_render_png ?? ''); + const datasetCaseDir = join(cfg.env.MUSE_ROOT ?? join(cfg.env.HOME ?? '', 'projects/muse'), 'data/muse/cases', caseName); + const museRoot = cfg.env.MUSE_ROOT ?? join(cfg.env.HOME ?? '', 'projects/muse'); + const pythonBin = cfg.env.MUSE_PYTHON ?? join(museRoot, '.venv/bin/python'); + + const judge = await judgeCase( + { + caseName, + datasetCaseDir, + candidatePng, + outPath: join(caseDir, 'judge.json'), + museRoot, + pythonBin, + baseUrl: cfg.baseUrl, + model: JUDGE_MODEL, + }, + stage12Ok && candidatePng.length > 0, + ); + writeState(caseDir, { + phase: 'judged', + attempts: state?.attempts ?? 1, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: result.score?.firstFailureCode ?? state?.firstFailureCode, + generationMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + return { + case: caseName, + status: 'ok', + phase: 'judged', + finalScore: judge.overall, + attempts: result.score?.attempts, + timeMs: result.score?.time_ms, + }; + } + + return { + case: caseName, + status: 'ok', + phase: 'scored', + finalScore: result.score?.score, + attempts: result.score?.attempts, + timeMs: result.score?.time_ms, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + writeState(caseDir, { + phase: 'infra_error', + attempts: state?.attempts ?? 0, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: state?.firstFailureCode, + generationMs: state?.generationMs, + error: message.slice(0, 1000), + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + return { case: caseName, status: 'infra', error: message }; + } +} + +async function main(): Promise { + const cfg = parseArgs(process.argv.slice(2)); + const preflightOnly = process.argv.includes('--preflight-only'); + + if (!preflightOnly) { + const report: PreflightReport = await runPreflight({ + env: cfg.env, + exists: existsSync, + run: (cmd, args) => + new Promise((res) => { + execFile(cmd, args, { timeout: 120_000 }, (err, stdout, stderr) => { + res({ + code: err ? ((err as { code?: number }).code ?? 1) : 0, + stdout: String(stdout ?? ''), + stderr: String(stderr ?? ''), + }); + }); + }), + fetchImpl: fetch, + caseCount: () => { + const dir = join(cfg.env.MUSE_ROOT ?? join(cfg.env.HOME ?? '', 'projects/muse'), 'data/muse/cases'); + return existsSync(dir) ? readdirSync(dir).length : 0; + }, + }); + console.log(formatPreflight(report)); + if (!report.ok) process.exit(1); + } + + const cases = discoverCases(cfg.cases); + if (cases.length === 0) { + console.error('No muse-* tasks found. Run importer first (Task 0).'); + process.exit(1); + } + mkdirSync(cfg.runRoot, { recursive: true }); + + const skillMd = buildSystemPrompt(cfg.skills); + const agent: AgentClient = cfg.mockFixture + ? new MockAgentClient(readCachedAgentResponses(cfg.mockFixture)) + : new OpenAICompatAgentClient({ + baseUrl: cfg.baseUrl, + apiKey: cfg.env.DEEPINFRA_API_KEY ?? '', + }); + + if (preflightOnly) { + console.log('preflight-only: OK'); + return; + } + + const totals = { tokensIn: 0, tokensOut: 0 }; + const startedAtMs = Date.now(); + writeFileSync( + join(cfg.runRoot, 'run.json'), + JSON.stringify( + { + runId: cfg.runId, + model: cfg.model, + baseUrl: cfg.baseUrl, + temperature: cfg.temperature, + maxAttempts: cfg.maxAttempts, + maxTokens: cfg.maxTokens, + skills: cfg.skills, + workers: cfg.workers, + protocol: PROTOCOL, + judgeModel: JUDGE_MODEL, + judgeBaseUrl: cfg.baseUrl, + startedAt: cfg.startedAt, + caseCount: cases.length, + }, + null, + 2, + ), + ); + + const outcomes = await mapPool(cases, cfg.workers, async (caseName, index) => { + const outcome = await runOneCase(caseName, cfg, agent, skillMd, totals); + const done = index + 1; + const badge = + outcome.status === 'ok' ? '✓' : outcome.status === 'infra' ? '✗' : '-'; + console.log( + `[${done}/${cases.length}] ${caseName} ${badge} ${ + outcome.finalScore !== undefined ? `final=${outcome.finalScore.toFixed(2)} ` : '' + }${outcome.error ?? outcome.status}`, + ); + return outcome; + }); + + const wallMs = Date.now() - startedAtMs; + const okCount = outcomes.filter((o) => o.status === 'ok').length; + const infra = outcomes.filter((o) => o.status === 'infra'); + writeFileSync( + join(cfg.runRoot, 'run.json'), + JSON.stringify( + { + runId: cfg.runId, + model: cfg.model, + baseUrl: cfg.baseUrl, + temperature: cfg.temperature, + maxAttempts: cfg.maxAttempts, + maxTokens: cfg.maxTokens, + skills: cfg.skills, + workers: cfg.workers, + protocol: PROTOCOL, + judgeModel: JUDGE_MODEL, + judgeBaseUrl: cfg.baseUrl, + startedAt: cfg.startedAt, + finishedAt: new Date().toISOString(), + caseCount: cases.length, + completed: okCount, + skipped: outcomes.filter((o) => o.status === 'skipped').length, + infraErrors: infra.length, + tokens: totals, + wallMs, + outcomes, + }, + null, + 2, + ), + ); + + writeReports(cfg.runRoot); + console.log(`\n${okCount}/${cases.length} complete in ${(wallMs / 60000).toFixed(1)} min; ${infra.length} infra errors`); + process.exit(infra.length > 0 ? 1 : 0); +} + +const isEntrypoint = (() => { + if (!process.argv[1]) return false; + try { + return import.meta.url === new URL(`file://${process.argv[1]}`).href; + } catch { + return false; + } +})(); + +if (isEntrypoint) { + main().catch((err) => { + console.error('Fatal:', err); + process.exit(1); + }); +} +``` + +Note: `readdirSync`/`TaskResult`/`CaseState` unused-import cleanup is part of Step 2. + +- [ ] **Step 2: Tighten imports and run a dry type pass** + +Run: `npx eslint scripts/runMuseSweep.ts` +Expected: 0 errors (fix unused imports flagged here). + +- [ ] **Step 3: Smoke the CLI against a fake run root with a mock fixture** + +Build a fixture from the committed stool solution (4 canned responses, enough for 2 cases with repairs): + +```bash +npx tsx -e " +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +const script = readFileSync('eval/tasks/muse-stool/solution-agent.kcad.ts', 'utf8'); +const text = ['\`\`\`ts', script, '\`\`\`'].join('\n'); +mkdirSync('eval/runs/golden-muse-sweep', { recursive: true }); +writeFileSync('eval/runs/golden-muse-sweep/fixture.json', JSON.stringify({ responses: Array.from({ length: 4 }, () => ({ text, tokens_in: 1, tokens_out: 1 })) }, null, 2)); +" +``` + +Then: + +```bash +export KERNELCAD_BIN=./dist/cli/index.js +export MUSE_ROOT=/home/andrii/projects/muse +export MUSE_PYTHON=/home/andrii/projects/muse/.venv/bin/python +npx tsx scripts/runMuseSweep.ts --cases stool chair --workers 2 \ + --mock-fixture eval/runs/golden-muse-sweep/fixture.json \ + --run-id _mock-muse --skip-judge +``` + +Expected: preflight passes; two case lines; `eval/runs/_mock-muse/run.json` + per-case `score.json`. + +- [ ] **Step 4: Commit** + +```bash +git add scripts/runMuseSweep.ts +git commit -m "eval: add resumable parallel MUSE sweep CLI" +``` + +--- + +### Task 10: Report generator + +**Files:** +- Create: `scripts/museReport.ts` +- Test: `scripts/museReport.test.ts` + +- [ ] **Step 1: Write the failing test** + +```ts +// scripts/museReport.test.ts +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { writeReports } from './museReport'; + +function makeCase( + root: string, + name: string, + sandboxOk: boolean, + overlapFree: boolean, + categories?: Record, +) { + const dir = join(root, 'cases', name); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'score.json'), JSON.stringify({ + gates: { 'evaluates clean': true }, + scored: {}, + gate_pass: true, score: 1, attempts: 2, + tokens: { input: 100, output: 20, total: 120 }, + time_ms: 1000, + metrics: { muse_sandbox_ok: sandboxOk, muse_overlap_free: overlapFree }, + })); + writeFileSync(join(dir, 'state.json'), JSON.stringify({ phase: 'judged', attempts: 2, tokens: { in: 100, out: 20 }, protocol: 'muse-v1', updatedAt: 'x' })); + if (categories) { + writeFileSync(join(dir, 'judge.json'), JSON.stringify({ overall: 0.5, categories, forcedZero: false })); + } +} + +const CATS = { + assembly_readiness: 1, joint_design: 0, tolerance: 1, + functional_adaptation: 1, usage_stability: 0, manufacturability: 1, +}; + +describe('writeReports', () => { + it('writes leaderboard json/csv, summary and protocol', () => { + const root = mkdtempSync(join(tmpdir(), 'report-')); + makeCase(root, 'a', true, true, CATS); + makeCase(root, 'b', false, false, CATS); + writeFileSync(join(root, 'run.json'), JSON.stringify({ model: 'm+kcad', judgeModel: 'judge', protocol: 'muse-v1', skills: ['kernelcad'], workers: 6, temperature: 0.2 })); + + writeReports(root); + + const row = JSON.parse(readFileSync(join(root, 'leaderboard.json'), 'utf8')); + expect(row.rows[0].cases).toBe(2); + expect(row.rows[0].judged).toBe(1); + expect(row.rows[0].final).toBe(33.33); + expect(readFileSync(join(root, 'summary.md'), 'utf8')).toContain('Forced-zero'); + expect(readFileSync(join(root, 'protocol.md'), 'utf8')).toContain('validator'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npx vitest run scripts/museReport.test.ts` +Expected: FAIL — module not found. + +- [ ] **Step 3: Implement** + +```ts +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + aggregateMuseSamples, + ZERO_CATEGORIES, + type JudgeCategories, + type MuseSample, +} from '../eval/lib/museAggregate'; +import type { Score } from '../eval/types'; + +export interface RunEnvelope { + runId?: string; + model: string; + judgeModel?: string; + baseUrl?: string; + protocol?: string; + skills?: string[]; + workers?: number; + temperature?: number; + caseCount?: number; + tokens?: { tokensIn?: number; tokensOut?: number }; + wallMs?: number; +} + +interface CaseArtifact { + case: string; + sandboxOk: boolean; + overlapFree: boolean; + categories?: JudgeCategories; + forcedZero: boolean; + score?: Score; + firstFailureCode?: string; +} + +function loadCase(root: string, name: string): CaseArtifact { + const dir = join(root, 'cases', name); + const score: Score | undefined = existsSync(join(dir, 'score.json')) + ? (JSON.parse(readFileSync(join(dir, 'score.json'), 'utf8')) as Score) + : undefined; + const metrics = (score?.metrics ?? {}) as Record; + let categories: JudgeCategories | undefined; + let forcedZero = false; + if (existsSync(join(dir, 'judge.json'))) { + const judge = JSON.parse(readFileSync(join(dir, 'judge.json'), 'utf8')) as { + categories?: JudgeCategories; + forcedZero?: boolean; + }; + categories = { ...ZERO_CATEGORIES, ...(judge.categories ?? {}) }; + forcedZero = judge.forcedZero === true; + } + return { + case: name, + sandboxOk: metrics.muse_sandbox_ok === true, + overlapFree: metrics.muse_overlap_free === true, + categories, + forcedZero, + score, + firstFailureCode: score?.firstFailureCode, + }; +} + +function toCsv(rows: Array>): string { + if (rows.length === 0) return ''; + const header = Object.keys(rows[0]); + const lines = [header.join(',')]; + for (const row of rows) { + lines.push(header.map((h) => String(row[h])).join(',')); + } + return `${lines.join('\n')}\n`; +} + +export function writeReports(runRoot: string): void { + const run: RunEnvelope = existsSync(join(runRoot, 'run.json')) + ? (JSON.parse(readFileSync(join(runRoot, 'run.json'), 'utf8')) as RunEnvelope) + : { model: 'unknown' }; + const casesDir = join(runRoot, 'cases'); + const names = existsSync(casesDir) ? readdirSync(casesDir).sort() : []; + const artifacts = names.map((n) => loadCase(runRoot, n)); + + const samples: MuseSample[] = artifacts.map((a) => ({ + case: a.case, + sandboxOk: a.sandboxOk, + overlapFree: a.overlapFree, + categories: a.categories, + })); + const { row, forcedZeroCases, judgedCases } = aggregateMuseSamples(samples, { + model: `${run.model} + kernelCAD`, + }); + + const leaderboard = { + judge: run.judgeModel ?? 'google/gemini-3.1-pro', + n_cases: artifacts.length, + updated: new Date().toISOString(), + validator_status: 'unpublished', + rows: [row], + }; + writeFileSync(join(runRoot, 'leaderboard.json'), JSON.stringify(leaderboard, null, 2)); + writeFileSync(join(runRoot, 'leaderboard.csv'), toCsv([row as unknown as Record])); + + const infra = artifacts.filter((a) => { + const statePath = join(runRoot, 'cases', a.case, 'state.json'); + if (!existsSync(statePath)) return false; + const state = JSON.parse(readFileSync(statePath, 'utf8')) as { phase?: string }; + return state.phase === 'infra_error'; + }); + const failureCounts = new Map(); + for (const a of artifacts) { + const code = a.firstFailureCode ?? 'none'; + failureCounts.set(code, (failureCounts.get(code) ?? 0) + 1); + } + const failureLines = [...failureCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([code, count]) => `- ${code}: ${count}`); + + const summary = [ + `# MUSE 106 sweep summary — ${run.model}`, + '', + `Cases: ${artifacts.length} | judged: ${judgedCases} | forced-zero: ${forcedZeroCases.length} | infra errors: ${infra.length}`, + `Sandbox pass: ${row.sandbox}% | Overlap-free: ${row.overlap_free}% | Final: ${row.final}`, + '', + '## Pillars', + '', + `- Functionality: ${row.functionality}`, + `- Manufacturability: ${row.manufacturability}`, + `- Assemblability: ${row.assemblability}`, + '', + '## First failure codes', + '', + ...(failureLines.length > 0 ? failureLines : ['- none']), + '', + '## Forced-zero cases', + '', + ...(forcedZeroCases.length > 0 ? forcedZeroCases.map((c) => `- ${c}`) : ['- none']), + '', + '## Infra errors', + '', + ...(infra.length > 0 ? infra.map((a) => `- ${a.case}`) : ['- none']), + '', + ].join('\n'); + writeFileSync(join(runRoot, 'summary.md'), summary); + + const protocol = [ + '# Sweep protocol and deviations', + '', + `- Driver: ${run.model}; protocol version ${run.protocol ?? 'muse-v1'}; temperature ${run.temperature ?? 0.2}; skills: ${(run.skills ?? []).join(', ') || 'n/a'}.`, + '- Generation: kernelCAD product loop — 1 sample per case, up to 2 diagnostic-driven repairs, candidates=1.', + '- Geometry submission: MUSE sandbox runs a CadQuery shim importing a kernelCAD-exported STEP (no CadQuery authored by kernelCAD).', + '- Judge: MUSE `generate_score_sp` + `_run_alignment_judge`, model google/gemini-3.1-pro served via DeepInfra (upstream default serving is OpenRouter preview).', + '- Candidate image: MUSE VTK render for all 106 cases; upstream uses DrawCAD 4-view PNGs for the 97 non-render-only cases (DrawCAD unpublished).', + '- Stage 2: MUSE external `validator` module is unpublished; watertight/manifold/self-intersection and the official `geom_valid` column cannot be computed locally. Overlap-free is computed with MUSE code. Forced-zero locally covers sandbox and overlap only.', + `- Aggregation mirrors upstream `generate_latex_tables_gemini.py@547a724^`; validator columns are null in leaderboard.json.`, + '', + ].join('\n'); + writeFileSync(join(runRoot, 'protocol.md'), protocol); +} +``` + +- [ ] **Step 4: Run the test** + +Run: `npx vitest run scripts/museReport.test.ts` +Expected: PASS. (Fix `require` usage to `readFileSync` imports before committing if eslint complains.) + +- [ ] **Step 5: Commit** + +```bash +git add scripts/museReport.ts scripts/museReport.test.ts +git commit -m "eval: add MUSE leaderboard/summary/protocol report generator" +``` + +--- + +### Task 11: Integration verification (mock replay, no API spend) + +- [ ] **Step 1: Full unit suite** + +Run: `npx vitest run eval/ scripts/museJudge.test.ts scripts/musePreflight.test.ts scripts/museReport.test.ts` +Expected: PASS. + +- [ ] **Step 2: Lint + typecheck** + +Run: `npm run lint && npm run typecheck` +Expected: PASS. (`eval/`/`scripts/` are outside tsconfig includes; eslint must still be clean.) + +- [ ] **Step 3: Mock replay through the real MUSE venv** + +```bash +export KERNELCAD_BIN=./dist/cli/index.js +export MUSE_ROOT=/home/andrii/projects/muse +export MUSE_PYTHON=/home/andrii/projects/muse/.venv/bin/python +npx tsx scripts/runMuseSweep.ts --cases stool chair --workers 2 \ + --mock-fixture eval/runs/golden-muse-sweep/fixture.json \ + --run-id _mock-muse-live --skip-judge +``` + +Expected: both cases produce `generated.step`, `muse/render/*.png`, and `score.json` with `metrics.muse_sandbox_ok: true`. + +- [ ] **Step 4: Resume test** + +Delete one case's `score.json` + set its `state.json` phase to `generated`, rerun the same command, and confirm only that case regenerates its score (the other reports `skipped`). + +- [ ] **Step 5: Commit any test fixture used** + +```bash +git add eval/runs/golden-muse-sweep/fixture.json +git commit -m "eval: add mock fixture for MUSE sweep integration test" +``` + +--- + +### Task 12: Live smoke (2 cases, real model + judge) + +- [ ] **Step 1: Source secrets and run two cases** + +```bash +set -a; source ~/.local/secrets/deepinfra.env; set +a +export KERNELCAD_BIN=./dist/cli/index.js +export MUSE_ROOT=/home/andrii/projects/muse +export MUSE_PYTHON=/home/andrii/projects/muse/.venv/bin/python +npx tsx scripts/runMuseSweep.ts --cases stool vase_teardrop --workers 2 --run-id smoke-2026-09-19 +``` + +Expected: preflight PASS; both cases judged; `leaderboard.json` written; `stool` sandbox+overlap pass. + +- [ ] **Step 2: Inspect artifacts** + +```bash +ls eval/runs/smoke-2026-09-19/cases/stool +cat eval/runs/smoke-2026-09-19/cases/stool/judge.json +cat eval/runs/smoke-2026-09-19/leaderboard.json +``` + +Expected: `judge.json` has six categories; leaderboard row has non-null pillars. + +- [ ] **Step 3: Verify transcript quality** + +Open `eval/runs/smoke-2026-09-19/cases/stool/transcript.md` and confirm turns, gate reports, and score sections are present. + +- [ ] **Step 4: Commit the fixture-free run note (no artifacts committed)** + +```bash +git status --porcelain # eval/runs/* is gitignored; must be clean except intended files +``` + +--- + +### Task 13: Full 106-case run + +- [ ] **Step 1: Launch the full run** + +```bash +set -a; source ~/.local/secrets/deepinfra.env; set +a +export KERNELCAD_BIN=./dist/cli/index.js +export MUSE_ROOT=/home/andrii/projects/muse +export MUSE_PYTHON=/home/andrii/projects/muse/.venv/bin/python +nohup npx tsx scripts/runMuseSweep.ts --workers 6 > /tmp/muse-sweep.log 2>&1 & +``` + +Expected: progress lines; target < 90 min. + +- [ ] **Step 2: Monitor and handle infra errors** + +```bash +tail -f /tmp/muse-sweep.log +``` + +On completion with infra errors, rerun the same command (resume reruns only `infra_error`/incomplete cases). + +- [ ] **Step 3: Review the report** + +```bash +cat eval/runs//summary.md +cat eval/runs//leaderboard.json +``` + +Expected: `final`, pillar values, forced-zero list, defect taxonomy. Review with Andrii before any external outreach. + +--- + +### Task 14: Documentation and PR + +**Files:** +- Create: `kernelCAD-private/docs/process/running-muse-benchmark.md` addendum (sweep section) — private repo +- Modify: `docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md` status → implemented + +- [ ] **Step 1: Add the sweep section to the private runbook** + +Append to `~/projects/kernelCAD-private/docs/process/running-muse-benchmark.md`: + +```markdown +## Full 106-case sweep (batch runner) + +The batch runner lives in kernelCAD-web: `scripts/runMuseSweep.ts`. + +```bash +set -a; source ~/.local/secrets/deepinfra.env; set +a +export KERNELCAD_BIN=./dist/cli/index.js +export MUSE_ROOT=~/projects/muse +export MUSE_PYTHON=~/projects/muse/.venv/bin/python +npx tsx scripts/runMuseSweep.ts --workers 6 +``` + +Outputs under `eval/runs/muse106---/`: per-case artifacts, +`leaderboard.json/csv`, `summary.md`, `protocol.md`. Resume is automatic; +`--force ` reruns a case; `touch /STOP` drains the pool. + +Known deviations are generated into `protocol.md` (validator unpublished, +judge serving, DrawCAD 4-view vs VTK render, skill-set trim). +``` + +- [ ] **Step 2: Update the spec status line** + +Change `Status: design approved …` to `Status: implemented (PR #)`. + +- [ ] **Step 3: Open the PR** + +```bash +git add docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md +git commit -m "docs: mark MUSE sweep spec implemented" +git push -u origin feat/muse-106-sweep +gh pr create --title "MUSE 106-case sweep runner (DeepSeek-V4.1-Flash + kernelCAD)" \ + --body "Implements docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md. Adds the batch sweep runner, DeepInfra agent client, MUSE judge wrapper, preflight, and report generator. Run artifacts and numbers are out of scope for the PR." +``` + +Expected: PR URL. + +--- + +## Self-review notes + +- Spec coverage: all spec sections map to tasks (client → T1, prompt → T2, state → T4, aggregate → T5, runner split → T6, judge → T7, preflight → T8, sweep → T9, reports → T10, speed/verification → T11–T13, docs → T14). Task import + venv prereqs → T0. +- Type consistency: `CaseState`/`MusePhase` used in T4/T9; `JudgeCategories` defined T5, used T7/T10; `writeReports(runRoot)` single-arg in T9/T10; `judgeCase` signature T7 matches T9 call site. +- No placeholder steps: all code blocks are complete; commands have expected output. diff --git a/docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md b/docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md new file mode 100644 index 000000000..6028d103d --- /dev/null +++ b/docs/superpowers/specs/2026-09-19-muse-106-sweep-design.md @@ -0,0 +1,294 @@ +# MUSE 106-Case Sweep — Open-Weight Driver + kernelCAD Agent Stack + +Status: implemented 2026-09-20. First full 106-case run complete — results and +gap analysis in `kernelCAD-private/docs/process/running-muse-benchmark.md` §7. +This spec is the source of truth for the implementation. Target: an official +MUSE leaderboard submission run. + +## Goal + +Run the full 106-case MUSE text-to-CAD benchmark (`dongxiaoyu/MUSE`, arXiv +2605.28579) with `deepseek-ai/DeepSeek-V4.1-Flash` (DeepInfra) driving the +kernelCAD agent stack — skills + gates + repair loop — and produce numbers, +artifacts, and a protocol document rigorous enough to become an official +MUSE leaderboard row. Wall-clock target: under ~90 minutes for all 106 cases +on the local 12-core box, versus days for the interactive pilot. + +## Decision record + +| Decision | Choice | +|---|---| +| Benchmark | MUSE, all 106 cases, 1 sample/case (their protocol) | +| Result usage | Official leaderboard submission; run first, negotiate package with maintainers after | +| Driver model | `deepseek-ai/DeepSeek-V4.1-Flash` via DeepInfra (OpenAI-compatible) | +| Protocol | kernelCAD product loop: 1 generation + up to 2 diagnostic-driven repairs; candidates=1 (no best-of-N) | +| Architecture | Dedicated batch runner (`scripts/runMuseSweep.ts`) reusing `runTask` internals | +| Judge | MUSE's own judge prompt + `generate_score_sp`; `google/gemini-3.1-pro` served via DeepInfra (preflight validates the exact model id; upstream/calibration defaults use OpenRouter preview serving — recorded as a deviation) | + +## Current state (gap analysis) + +Exists: +- `eval/oracle/museScorer.ts` + `museScorerWrapper.py` — pushes a kernelCAD + STEP through MUSE's own sandbox, interpenetration check, and VTK render. + Pilot-proven on 10 cases. +- `eval/tasks/muse-*` for 10 pilot cases; `eval/lib/importMuseTasks.ts` + generates `prompt.md` + `harness.ts` from the dataset (idempotent). +- `eval/runner.ts` `runTask()` — closed loop (generate → gate → repair) + + harness scoring + `score.json`/`transcript.md`; `candidates=1` already + supported. +- MUSE checkout at `~/projects/muse` with all 106 cases of the HF dataset + under `data/muse/cases/`; a full qwen-2.5-72b calibration run under + `out/calib106_qwen72b/` (reproduces published stage-1 rate). + +Missing: +- 96 of 106 MUSE tasks not imported. +- No non-Anthropic `AgentClient` (DeepInfra adapter). +- No batch orchestration, resume, aggregation, or judge batching. +- MUSE Python venv not present on this machine (cadquery + vtk). +- `loadCombinedSkillMd()` concatenates all 18 skills (300,511 bytes ≈ 75k + tokens/turn) — unusable for a 106-case sweep. + +## Non-goals + +- No bare-model CadQuery run (existing leaderboard rows already cover that). +- No multi-model comparison in this run. +- No benchmark-specific prompt hints beyond the documented retarget to + `.kcad.ts` (same adaptation contract as the cqe importer). +- No gate/scorer tuning to fit the benchmark; MUSE thresholds and code only. +- No changes to the interactive skills' content for this run. + +## Architecture + +New files, all in `kernelCAD-web`: + +| File | Responsibility | +|---|---| +| `eval/agentOpenAICompat.ts` | `OpenAICompatAgentClient implements AgentClient`. Global `fetch`, no new npm deps. Retries 429/5xx/timeout with exponential backoff + jitter. No Anthropic `cache_control` blocks. | +| `eval/lib/systemPrompt.ts` | `buildSystemPrompt(skillDirs: string[]): string` — reads selected `SKILL.md` files, sorted, joined with `---`. Default selection for the sweep: `kernelcad`, `kernelcad-authoring`, `kernelcad-assemblies`, `kernelcad-parts` (pilot parity). | +| `scripts/runMuseSweep.ts` | Batch entry point: CLI parsing, preflight, worker pool, resume, progress, run envelope, budget guard, STOP-file drain. | +| `eval/oracle/museJudgeWrapper.py` | Calls MUSE's own judge (`generate_score_sp` prompt + `_run_alignment_judge`) for one sample; applies funnel forced-zero; emits `judge.json`. | +| `scripts/museJudge.ts` | Parallel judge pass over `scored` samples (pool, retries, per-case `judge.json`). | +| `scripts/museReport.ts` | Aggregates all cases into `leaderboard.json/csv`, `summary.md`, `protocol.md`. | +| `scripts/musePreflight.ts` | Env/asset verification + one no-agent scorer run on a known STEP. | + +Refactor in `eval/runner.ts` (behavior-preserving): +split `runTask` into +- `generateCase(args)` → runs the closed loop, writes `output.kcad.ts`, + returns loop stats; sets `state.phase = 'generated'`. Accepts + `maxAttempts`, `maxTokens`, and an explicit `temperature` (today + `MAX_ATTEMPTS=3`, `MAX_TOKENS=8000` and `variantTemperature()` are + hardcoded in the runner; the sweep threads its CLI values through). +- `scoreCase(args)` → final evaluate + harness + `score.json`; sets + `state.phase = 'scored'`. +- `runTask(args)` → compose both (existing callers unchanged; existing + tests must stay green). + +Resume needs `scoreCase` without `generateCase`, which is why the split is +required. + +### Interfaces + +```ts +// eval/agentOpenAICompat.ts +export interface OpenAICompatOptions { + baseUrl: string; // default https://api.deepinfra.com/v1/openai + apiKey: string; + maxRetries?: number; // default 5 + retryBaseMs?: number; // default 1000 + retryMaxMs?: number; // default 30000 + fetchImpl?: typeof fetch; // tests +} +``` + +```ts +// scripts/runMuseSweep.ts (flags) +--cases default: all imported muse-* tasks +--workers default 6 +--model default deepseek-ai/DeepSeek-V4.1-Flash +--base-url default https://api.deepinfra.com/v1/openai +--temperature default 0.2 +--max-attempts default 3 (1 generation + 2 repairs) +--skills default the 4 above +--run-id default muse106--- +--force ignore existing state for these cases +--skip-judge target phase = scored instead of judged +--max-tokens-in budget guard, default 25_000_000 +--mock-fixture mock agent replay for integration tests +--preflight-only +``` + +## Data flow per case + +1. `prompt.md` (dataset `design_description.md`, retargeted to `.kcad.ts`) → + DeepSeek-V4.1-Flash; system = trimmed skills; temp 0.2; max_tokens 8000. +2. `extractScript` → write `output.kcad.ts` → gates: `kernelcad evaluate + --json` + `kernelcad interference --json` (existing `webGateRunner`). +3. Gate failure → diagnostic-driven repair prompt, up to 2 repairs + (existing closed loop; `candidates=1`). +4. `kernelcad export step` → `museScorerWrapper.py` with + `MUSE_PYTHON=/.venv/bin/python` (direct venv, no `uv run`): + MUSE sandbox execution, interpenetration check, VTK render. +5. Harness scores stage 1 + overlap; `score.json` written. +6. Judge phase: `museJudgeWrapper.py` (MUSE venv python), candidate = MUSE's + VTK render, reference = `.png` (or `_stp_render.png` for cases + listed in `render_only_cases.txt`); temp 0.1, n=1; forced-zero on stage + 1/overlap failure; `judge.json` written. +7. Report phase: aggregate only (no re-computation of scores). + +## State machine and resume + +`cases//state.json`: + +```json +{ "phase": "pending|generated|scored|judged|infra_error", + "attempts": 2, "tokens": { "in": 0, "out": 0 }, + "firstFailureCode": "eval.no-script-extracted", + "protocol": "muse-v1", "updatedAt": "..." } +``` + +Resume rules (resume is always on; `--force ` is the override): +- Target phase is `judged`, or `scored` when `--skip-judge` is set. Cases at + or past the target phase are skipped. +- `generated` but scoring crashed → reuse `output.kcad.ts`, do not re-call + the model. +- `judged` is terminal for a sweep invocation; `--force ` restarts it. +- Infra errors are retried on the next invocation automatically. + +## Error handling, budgets, concurrency + +- Retries: agent HTTP 5 (backoff 1s→30s, jitter); CLI gate 2; scorer 2; + judge 5. Only transport/5xx/429 errors are retried — model-level failures + are legitimate low scores, never retried. +- Timeouts: agent request 180s; gate 300s; scorer 300s; judge 180s. +- One case never kills the pool. Infra failures are excluded from aggregate + denominators but always listed. +- Budget guard: cumulative input-token cap (`--max-tokens-in`); checked at + phase boundaries; abort leaves a resumable run. +- Kill switch: `touch /STOP` → pool drains, no new cases start. +- Exit codes: 0 = all cases at target phase; 1 = infra failures present. +- Concurrency: 6 cases in parallel default; agent calls within a case are + sequential. Lower to 4 if python stages OOM (14 GB box). + +## Outputs and packaging + +``` +eval/runs// + run.json # git SHA (+dirty flag), model, temp, skills, workers, + # protocol version, judge model/serving, totals, timings + protocol.md # MUSE protocol mapping + every deviation (generated) + cases// + output.kcad.ts + transcript.md + score.json # existing Score shape + metrics.muse_* + state.json + generated.step + muse/ # code.py shim, render PNG/STL/STEP, geometry payload + judge.json # categories + overall + forced_zero reason + leaderboard.json + leaderboard.csv + summary.md +``` + +`leaderboard.json` uses MUSE's column names +(`sandbox`, `overlap_free`, `functionality`, `manufacturability`, +`assemblability`, `final`, six category columns). Validator-dependent +columns (`watertight`, `manifold`, `self_int_free`, `geom_valid`) are +`null` with `"validator_status": "unpublished"` — never fabricated. + +Aggregation arithmetic (upstream definition, verified against the deleted +`scripts/bench_evaluate/generate_latex_tables_gemini.py` at MUSE commit +`547a724^`; pin the reference in the aggregator source): + +| Pillar | Definition | +|---|---| +| Functionality | mean(Functional Adaptation, Usage Stability) | +| Manufacturability | mean(Tolerance, Manufacturability) | +| Assemblability | mean(Assembly Readiness, Joint Design) | +| Final | mean(Functionality, Manufacturability, Assemblability) | + +Forced zero: any stage-1 or stage-2 failure zeroes all six categories. +Locally, stage 2 covers only `sandbox` + `overlap_free`; upstream's rule +also zeroes on the unpublished validator's `watertight`/`manifold`/ +`self_int_free` failures — `protocol.md` must state this gap. + +`summary.md`: funnel attrition (counts per stage), defect taxonomy grouped +by `firstFailureCode`, tokens/cost/wall-clock, infra-error list. + +## Prerequisites and environment + +1. `npm run build:cli` in `kernelCAD-web`; `KERNELCAD_BIN=./dist/cli/index.js`. +2. MUSE venv (missing on this box): + `cd ~/projects/muse && uv venv .venv --python 3.12 && uv pip install -e .` + (cadquery + vtk). Set `MUSE_ROOT`/`MUSE_PYTHON` for the oracle. +3. Dataset present at `~/projects/muse/data/muse/cases/` (106 cases — yes). +4. `DEEPINFRA_API_KEY` from `~/.local/secrets/deepinfra.env` (sourced, never + written into artifacts). +5. Import remaining 96 tasks via `importMuseTasks.ts` and commit all 106 + `eval/tasks/muse-*` dirs (prompt snapshots = reproducibility). +6. Working tree: implement in a fresh worktree off `develop` (the current + checkout is a stale hotfix branch); pin the SHA in `run.json`. + +## Speed levers (vs the pilot) + +| Lever | Saving | +|---|---| +| Trimmed 4-skill prompt (142 KB ≈ 35k tokens vs 300 KB ≈ 75k for all 18) | smaller prefill on every turn | +| `candidates=1` (the runner's default; only `eval/run.ts` sets `BEST_OF_N=4`) | 4× fewer agent calls than the batch eval default, 1× vs protocol | +| 6-case parallel pool | ~6× wall-clock | +| Direct venv python instead of `uv run` per case | ~2–4s × 106 scorer startups | +| Batched/parallel judge | minutes vs sequential hours | +| Resume checkpoints | crashed runs continue instead of restarting | + +Not doing: batching MUSE's Python stages into a persistent worker (complexity +justified only if smoke shows scorer startup dominating). + +## Testing and verification + +- Unit (vitest): OpenAI-compat client (mock fetch: success, 429→retry, + bad JSON, timeout), `buildSystemPrompt` selection, resume state machine, + aggregator arithmetic vs fixture (forced-zero, `final` mean), `run.json` + schema. +- Integration (no API): `--mock-fixture` replay of 2 cases through the real + MUSE venv, asserting artifacts and `score.json` shape. +- Preflight: `--preflight-only` verifies keys, venv imports, CLI, task + count, and runs the scorer on a calibration STEP + (`out/calib106_qwen72b/.../business_card_holder...step`). +- Live smoke: 2 cases with the real model, then full 106. +- Sanity: stage-1 sandbox rate vs the qwen calibration run (plumbing check, + not accuracy). +- Gates before merge: `npm run lint`, `npm run typecheck`, `npm test`. + +## Acceptance criteria + +1. All 106 tasks imported; importer re-run is a no-op diff. +2. Full run completes unattended with `--workers 6`; kill/rerun resumes + without regenerating completed cases. +3. Every case has the artifact set above; infra failures are listed with + retry evidence and excluded from denominators. +4. `leaderboard.json` arithmetic matches upstream definitions + (unit-tested); `protocol.md` lists every deviation. +5. Lint, typecheck, unit + integration tests green; no secrets in repo or + artifacts. +6. Numbers reviewed by Andrii before any external outreach. + +## Risks and open questions (for maintainer negotiation) + +1. External `validator` module is unpublished (their GitHub issue #3): + official `geom_valid` cannot be computed locally. We report overlap-free + + kernelCAD watertight export evidence and ask maintainers to run their + validator on the submitted package. +2. Judge serving: DeepInfra serves the same model family as the paper's + OpenRouter preview; serving differs. Documented in `protocol.md`; + consider an OpenRouter re-run if a key is available. +3. Leaderboard row naming for a tool-assisted entry + (`deepseek-v4.1-flash + kernelCAD`) — to agree with maintainers. +4. Prompt trim to 4 skills changes nothing semantically but is a protocol + difference vs the interactive pilot; stated in `protocol.md`. +5. Judge candidate image: upstream uses DrawCAD 4-view PNGs for the 97 + non-render-only cases; DrawCAD is unpublished here, so all 106 use the + MUSE VTK render (same as the 9 render-only cases). Stated in + `protocol.md`; ask maintainers for the DrawCAD path at negotiation time. +6. Parallel python stages may pressure memory; workers knob documented. +7. `eval/` and `scripts/` are outside `tsconfig.app/node` includes, so + `npm run typecheck` doesn't cover the new files; verification relies on + eslint + vitest, with an optional tsconfig include as part of the plan. diff --git a/eval/.gitignore b/eval/.gitignore index 3285bdc69..4b1615d18 100644 --- a/eval/.gitignore +++ b/eval/.gitignore @@ -1,3 +1,7 @@ # Live runs are not committed; only golden fixtures (runs/golden-*) are. runs/* !runs/golden-*/ + +# Python bytecode from eval/oracle wrappers +__pycache__/ +*.pyc diff --git a/eval/agentOpenAICompat.test.ts b/eval/agentOpenAICompat.test.ts new file mode 100644 index 000000000..be0f1be40 --- /dev/null +++ b/eval/agentOpenAICompat.test.ts @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { describe, expect, it, vi } from 'vitest'; +import { OpenAICompatAgentClient } from './agentOpenAICompat'; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +const REQ = { + system: 'S', + messages: [{ role: 'user' as const, content: 'hi' }], + model: 'deepseek-ai/DeepSeek-V4.1-Flash', + max_tokens: 100, + temperature: 0.2, +}; + +describe('OpenAICompatAgentClient', () => { + it('returns text and usage from a successful completion', async () => { + const fetchImpl = vi.fn(async () => + jsonResponse({ + choices: [{ message: { content: 'hello' } }], + usage: { prompt_tokens: 11, completion_tokens: 7 }, + }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1/', + apiKey: 'k', + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out).toEqual({ text: 'hello', tokens_in: 11, tokens_out: 7 }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe('https://api.example.com/v1/chat/completions'); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.model).toBe('deepseek-ai/DeepSeek-V4.1-Flash'); + expect(body.messages[0]).toEqual({ role: 'system', content: 'S' }); + expect(body.temperature).toBe(0.2); + }); + + it('retries 429 then succeeds', async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response('rate', { status: 429 })) + .mockResolvedValueOnce( + jsonResponse({ choices: [{ message: { content: 'ok' } }], usage: {} }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out.text).toBe('ok'); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it('retries a network error up to the cap, then throws', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + maxRetries: 2, + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + await expect(client.generate(REQ)).rejects.toThrow('ECONNRESET'); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it('retries a bad JSON body then succeeds', async () => { + const badJson = { + status: 200, + ok: true, + json: async () => { + throw new SyntaxError('Unexpected token < in JSON'); + }, + } as unknown as Response; + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(badJson) + .mockResolvedValueOnce( + jsonResponse({ choices: [{ message: { content: 'ok' } }], usage: {} }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out.text).toBe('ok'); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it('retries a timeout error then succeeds', async () => { + const fetchImpl = vi + .fn() + .mockRejectedValueOnce( + new DOMException('The operation was aborted due to timeout', 'TimeoutError'), + ) + .mockResolvedValueOnce( + jsonResponse({ choices: [{ message: { content: 'ok' } }], usage: {} }), + ); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + const out = await client.generate(REQ); + expect(out.text).toBe('ok'); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it('throws on a non-retryable 400 without retrying', async () => { + const fetchImpl = vi.fn(async () => new Response('bad model', { status: 400 })); + const client = new OpenAICompatAgentClient({ + baseUrl: 'https://api.example.com/v1', + apiKey: 'k', + retryBaseMs: 1, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + await expect(client.generate(REQ)).rejects.toThrow('HTTP 400'); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); diff --git a/eval/agentOpenAICompat.ts b/eval/agentOpenAICompat.ts new file mode 100644 index 000000000..6b7ed0b1b --- /dev/null +++ b/eval/agentOpenAICompat.ts @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import type { AgentClient, AgentMessage, AgentResponse } from './types'; + +export interface OpenAICompatOptions { + baseUrl: string; + apiKey: string; + /** Total attempts = maxRetries + 1. Default 5 (6 attempts). */ + maxRetries?: number; + retryBaseMs?: number; + retryMaxMs?: number; + fetchImpl?: typeof fetch; +} + +interface ChatCompletionResponse { + choices?: Array<{ message?: { content?: string | null } }>; + usage?: { prompt_tokens?: number; completion_tokens?: number }; +} + +const RETRYABLE_STATUS = (status: number): boolean => status === 429 || status >= 500; + +const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0); + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export class OpenAICompatAgentClient implements AgentClient { + private readonly baseUrl: string; + private readonly apiKey: string; + private readonly maxRetries: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly fetchImpl: typeof fetch; + + constructor(opts: OpenAICompatOptions) { + this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); + this.apiKey = opts.apiKey; + this.maxRetries = opts.maxRetries ?? 5; + this.retryBaseMs = opts.retryBaseMs ?? 1000; + this.retryMaxMs = opts.retryMaxMs ?? 30000; + this.fetchImpl = opts.fetchImpl ?? fetch; + } + + async generate(args: { + system: string; + systemAddendum?: string; + messages: AgentMessage[]; + model: string; + max_tokens: number; + temperature?: number; + }): Promise { + const system = + args.systemAddendum && args.systemAddendum.length > 0 + ? `${args.system}\n\n${args.systemAddendum}` + : args.system; + const body = { + model: args.model, + max_tokens: args.max_tokens, + messages: [ + { role: 'system', content: system }, + ...args.messages.map((m) => ({ role: m.role, content: m.content })), + ], + ...(args.temperature !== undefined ? { temperature: args.temperature } : {}), + }; + + let lastErr: Error | undefined; + for (let attempt = 0; attempt <= this.maxRetries; attempt++) { + if (attempt > 0) { + const backoff = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** (attempt - 1)); + await sleep(backoff + Math.random() * backoff * 0.25); + } + let resp: Response; + try { + resp = await this.fetchImpl(`${this.baseUrl}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(180_000), + }); + } catch (err) { + lastErr = err instanceof Error ? err : new Error(String(err)); + continue; + } + if (RETRYABLE_STATUS(resp.status)) { + const text = await resp.text().catch(() => ''); + lastErr = new Error(`HTTP ${resp.status} ${text.slice(0, 300)}`); + continue; + } + if (!resp.ok) { + const text = await resp.text().catch(() => ''); + throw new Error(`OpenAI-compat request failed: HTTP ${resp.status} ${text.slice(0, 300)}`); + } + let data: ChatCompletionResponse; + try { + data = (await resp.json()) as ChatCompletionResponse; + } catch (err) { + lastErr = err instanceof Error ? err : new Error(String(err)); + continue; + } + const text = data.choices?.[0]?.message?.content ?? ''; + if (text.length === 0) { + return { text: '', tokens_in: num(data.usage?.prompt_tokens), tokens_out: 0 }; + } + return { + text, + tokens_in: num(data.usage?.prompt_tokens), + tokens_out: num(data.usage?.completion_tokens), + }; + } + throw lastErr ?? new Error('OpenAI-compat request failed after retries'); + } +} diff --git a/eval/lib/importMuseTasks.ts b/eval/lib/importMuseTasks.ts index 57b41f2de..743433942 100644 --- a/eval/lib/importMuseTasks.ts +++ b/eval/lib/importMuseTasks.ts @@ -47,7 +47,9 @@ ${designDescription.trim()} } function harnessFor(caseName: string): string { - return `// eval/tasks/muse-${caseName}/harness.ts + return `// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-${caseName}/harness.ts // // Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark // case '${caseName}' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). diff --git a/eval/lib/museAggregate.test.ts b/eval/lib/museAggregate.test.ts new file mode 100644 index 000000000..3a659bfaa --- /dev/null +++ b/eval/lib/museAggregate.test.ts @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { describe, expect, it } from 'vitest'; +import { aggregateMuseSamples, type MuseSample } from './museAggregate'; + +const judged = (overrides: Partial = {}): MuseSample => ({ + case: 'c', + sandboxOk: true, + overlapFree: true, + categories: { + assembly_readiness: 1, + joint_design: 0, + tolerance: 1, + functional_adaptation: 1, + usage_stability: 0, + manufacturability: 1, + }, + ...overrides, +}); + +describe('aggregateMuseSamples', () => { + it('maps the six categories to the three pillars and final', () => { + const { row, forcedZeroCases, judgedCases } = aggregateMuseSamples([judged()], { + model: 'm+kcad', + }); + expect(row.functional).toBe(100); + expect(row.robust).toBe(0); + expect(row.functionality).toBe(50); + expect(row.well_toleranced).toBe(100); + expect(row.manufacturable).toBe(100); + expect(row.manufacturability).toBe(100); + expect(row.assembly_ready).toBe(100); + expect(row.connectable).toBe(0); + expect(row.assemblability).toBe(50); + expect(row.final).toBe(66.67); + expect(row.sandbox).toBe(100); + expect(row.overlap_free).toBe(100); + expect(row.watertight).toBeNull(); + expect(row.manifold).toBeNull(); + expect(row.self_int_free).toBeNull(); + expect(row.geom_valid).toBeNull(); + expect(row.judged).toBe(1); + expect(row.cases).toBe(1); + expect(judgedCases).toBe(1); + expect(forcedZeroCases).toEqual([]); + }); + + it('zeroes all categories when stage 1 or overlap fails', () => { + const { row, forcedZeroCases, judgedCases } = aggregateMuseSamples( + [ + judged({ case: 'stage1-fail', sandboxOk: false }), + judged({ case: 'overlap-fail', overlapFree: false, categories: judged().categories }), + ], + { model: 'm+kcad' }, + ); + expect(row.final).toBe(0); + expect(row.robust).toBe(0); + expect(row.well_toleranced).toBe(0); + expect(row.sandbox).toBe(50); + expect(row.overlap_free).toBe(0); + expect(forcedZeroCases).toEqual(['stage1-fail', 'overlap-fail']); + expect(judgedCases).toBe(0); + }); + + it('averages across cases and counts judged samples', () => { + const { row, forcedZeroCases, judgedCases } = aggregateMuseSamples( + [judged({ case: 'ok' }), judged({ case: 'no-sandbox', sandboxOk: false })], + { model: 'm+kcad' }, + ); + expect(row.cases).toBe(2); + expect(row.judged).toBe(1); + expect(row.sandbox).toBe(50); + expect(row.final).toBe(33.33); + expect(judgedCases).toBe(1); + expect(forcedZeroCases).toEqual(['no-sandbox']); + }); + + it('keeps unjudged samples in the denominator without forcing zero', () => { + const { row, forcedZeroCases, judgedCases, infraCases } = aggregateMuseSamples( + [judged({ case: 'unjudged', categories: undefined })], + { model: 'm+kcad' }, + ); + expect(row.cases).toBe(1); + expect(row.sandbox).toBe(100); + expect(row.judged).toBe(0); + expect(row.final).toBe(0); + expect(judgedCases).toBe(0); + expect(forcedZeroCases).toEqual([]); + expect(infraCases).toEqual([]); + }); + + it('excludes infra errors from denominators and reports them', () => { + const { row, forcedZeroCases, judgedCases, infraCases } = aggregateMuseSamples( + [judged({ case: 'good' }), judged({ case: 'bad', infra: true, sandboxOk: false })], + { model: 'm+kcad' }, + ); + expect(row.cases).toBe(1); + expect(row.sandbox).toBe(100); + expect(row.final).toBe(66.67); + expect(row.judged).toBe(1); + expect(judgedCases).toBe(1); + expect(forcedZeroCases).toEqual([]); + expect(infraCases).toEqual(['bad']); + }); + + it('returns zeroed rates for empty input', () => { + const { row, forcedZeroCases, judgedCases, infraCases } = aggregateMuseSamples([], { + model: 'm+kcad', + }); + expect(row.cases).toBe(0); + expect(row.judged).toBe(0); + expect(row.sandbox).toBe(0); + expect(row.overlap_free).toBe(0); + expect(row.final).toBe(0); + expect(judgedCases).toBe(0); + expect(forcedZeroCases).toEqual([]); + expect(infraCases).toEqual([]); + }); +}); diff --git a/eval/lib/museAggregate.ts b/eval/lib/museAggregate.ts new file mode 100644 index 000000000..76fcd9f22 --- /dev/null +++ b/eval/lib/museAggregate.ts @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// +// Aggregation mirrors MUSE's leaderboard arithmetic +// (`scripts/bench_evaluate/generate_latex_tables_gemini.py` at commit 547a724^): +// functionality = mean(functional_adaptation, usage_stability) +// manufacturability = mean(tolerance, manufacturability) +// assemblability = mean(assembly_readiness, joint_design) +// final = mean(functionality, manufacturability, assemblability) +// Any stage-1/stage-2 failure zeroes all six categories. + +export interface JudgeCategories { + assembly_readiness: number; + joint_design: number; + tolerance: number; + functional_adaptation: number; + usage_stability: number; + manufacturability: number; +} + +export interface MuseSample { + case: string; + sandboxOk: boolean; + overlapFree: boolean; + /** Judge categories (0/1); absent when the judge was not run. */ + categories?: JudgeCategories; + /** + * Infra-error cases are excluded from all aggregate denominators and + * returned separately as `infraCases`. + */ + infra?: boolean; +} + +export interface MuseLeaderboardRow { + model: string; + /** + * Samples with effective categories (forced-zero and infra cases excluded). + * As a count, this is not a percentage; every other numeric column is 0–100. + */ + judged: number; + /** Non-infra samples, i.e. the denominator of every percentage column. */ + cases: number; + sandbox: number; + overlap_free: number; + /** Validator-only column: always null locally (upstream validator unpublished). */ + watertight: number | null; + /** Validator-only column: always null locally (upstream validator unpublished). */ + manifold: number | null; + /** Validator-only column: always null locally (upstream validator unpublished). */ + self_int_free: number | null; + /** Validator-only column: always null locally (upstream validator unpublished). */ + geom_valid: number | null; + functionality: number; + manufacturability: number; + assemblability: number; + final: number; + functional: number; + robust: number; + well_toleranced: number; + manufacturable: number; + assembly_ready: number; + connectable: number; +} + +export const ZERO_CATEGORIES: Readonly = { + assembly_readiness: 0, + joint_design: 0, + tolerance: 0, + functional_adaptation: 0, + usage_stability: 0, + manufacturability: 0, +}; + +const pct = (x: number): number => Math.round(x * 10000) / 100; + +/** Effective categories for one sample after the funnel forced-zero rule. */ +export function effectiveCategories(sample: MuseSample): JudgeCategories | null { + if (!sample.sandboxOk || !sample.overlapFree) return null; + return sample.categories ?? null; +} + +function samplePillars(categories: JudgeCategories): { + functional: number; + robust: number; + well_toleranced: number; + manufacturable: number; + assembly_ready: number; + connectable: number; + functionality: number; + manufacturability: number; + assemblability: number; + final: number; +} { + const functionality = + (categories.functional_adaptation + categories.usage_stability) / 2; + const manufacturability = (categories.tolerance + categories.manufacturability) / 2; + const assemblability = (categories.assembly_readiness + categories.joint_design) / 2; + return { + functional: categories.functional_adaptation, + robust: categories.usage_stability, + well_toleranced: categories.tolerance, + manufacturable: categories.manufacturability, + assembly_ready: categories.assembly_readiness, + connectable: categories.joint_design, + functionality, + manufacturability, + assemblability, + final: (functionality + manufacturability + assemblability) / 3, + }; +} + +export function aggregateMuseSamples( + samples: readonly MuseSample[], + meta: { model: string }, +): { + row: MuseLeaderboardRow; + forcedZeroCases: string[]; + judgedCases: number; + infraCases: string[]; +} { + const counted = samples.filter((s) => !s.infra); + const infraCases = samples.filter((s) => s.infra).map((s) => s.case); + // Empty and all-infra inputs would otherwise divide by zero. + const cases = counted.length || 1; + const forcedZeroCases: string[] = []; + let sandboxPass = 0; + let overlapPass = 0; + let judged = 0; + const sums = { + functionality: 0, + manufacturability: 0, + assemblability: 0, + final: 0, + functional: 0, + robust: 0, + well_toleranced: 0, + manufacturable: 0, + assembly_ready: 0, + connectable: 0, + }; + + for (const sample of counted) { + if (sample.sandboxOk) sandboxPass++; + if (sample.sandboxOk && sample.overlapFree) overlapPass++; + const effective = effectiveCategories(sample); + if (!effective) { + if (sample.categories) forcedZeroCases.push(sample.case); + continue; + } + judged++; + const pillars = samplePillars(effective); + sums.functional += pillars.functional; + sums.robust += pillars.robust; + sums.well_toleranced += pillars.well_toleranced; + sums.manufacturable += pillars.manufacturable; + sums.assembly_ready += pillars.assembly_ready; + sums.connectable += pillars.connectable; + sums.functionality += pillars.functionality; + sums.manufacturability += pillars.manufacturability; + sums.assemblability += pillars.assemblability; + sums.final += pillars.final; + } + + const row: MuseLeaderboardRow = { + model: meta.model, + judged, + cases: counted.length, + sandbox: pct(sandboxPass / cases), + overlap_free: pct(overlapPass / cases), + watertight: null, + manifold: null, + self_int_free: null, + geom_valid: null, + functionality: pct(sums.functionality / cases), + manufacturability: pct(sums.manufacturability / cases), + assemblability: pct(sums.assemblability / cases), + final: pct(sums.final / cases), + functional: pct(sums.functional / cases), + robust: pct(sums.robust / cases), + well_toleranced: pct(sums.well_toleranced / cases), + manufacturable: pct(sums.manufacturable / cases), + assembly_ready: pct(sums.assembly_ready / cases), + connectable: pct(sums.connectable / cases), + }; + return { row, forcedZeroCases, judgedCases: judged, infraCases }; +} diff --git a/eval/lib/museState.test.ts b/eval/lib/museState.test.ts new file mode 100644 index 000000000..509f8340e --- /dev/null +++ b/eval/lib/museState.test.ts @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { isAtLeast, readState, writeState, type CaseState } from './museState'; + +const BASE: Omit = { + phase: 'generated', + attempts: 2, + tokens: { in: 100, out: 20 }, + protocol: 'muse-v1', +}; + +describe('museState', () => { + it('writes and reads state', () => { + const dir = mkdtempSync(join(tmpdir(), 'mstate-')); + const first: CaseState = { + ...BASE, + firstFailureCode: 'x', + generationMs: 42, + updatedAt: '2026-09-19T00:00:00Z', + }; + writeState(dir, first); + expect(readState(dir)).toEqual(first); + + const second: CaseState = { + ...BASE, + phase: 'scored', + updatedAt: '2026-09-19T00:01:00Z', + }; + writeState(dir, second); + expect(readState(dir)).toEqual(second); + }); + + it('returns null when no state file exists', () => { + const dir = mkdtempSync(join(tmpdir(), 'mstate-')); + expect(readState(dir)).toBeNull(); + }); + + it('returns null for corrupt JSON', () => { + const dir = mkdtempSync(join(tmpdir(), 'mstate-')); + writeFileSync(join(dir, 'state.json'), '{oops'); + expect(readState(dir)).toBeNull(); + }); + + it('orders phases and treats infra_error as never at-target', () => { + expect(isAtLeast('judged', 'scored')).toBe(true); + expect(isAtLeast('generated', 'scored')).toBe(false); + expect(isAtLeast('pending', 'generated')).toBe(false); + expect(isAtLeast('infra_error', 'pending')).toBe(false); + expect(isAtLeast('scored', 'scored')).toBe(true); + expect(isAtLeast('judged', 'infra_error')).toBe(false); + }); +}); diff --git a/eval/lib/museState.ts b/eval/lib/museState.ts new file mode 100644 index 000000000..aa82434ef --- /dev/null +++ b/eval/lib/museState.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export type MusePhase = 'pending' | 'generated' | 'scored' | 'judged' | 'infra_error'; + +export interface CaseState { + phase: MusePhase; + attempts: number; + tokens: { in: number; out: number }; + firstFailureCode?: string; + /** Generation wall-clock, preserved so resumed scoring can report totals. */ + generationMs?: number; + error?: string; + protocol: string; + updatedAt: string; +} + +const PHASE_ORDER: Record, number> = { + pending: 0, + generated: 1, + scored: 2, + judged: 3, +}; + +export function statePath(caseDir: string): string { + return join(caseDir, 'state.json'); +} + +/** + * Writes state atomically: the JSON goes to a sibling `.tmp` file and is then + * renamed over `state.json`, so concurrent readers observe either the previous + * or the new state and a crash mid-write cannot corrupt resume state. No + * fsync, so this is not durable against power loss. + */ +export function writeState(caseDir: string, state: CaseState): void { + const target = statePath(caseDir); + const tmp = `${target}.tmp`; + writeFileSync(tmp, JSON.stringify(state, null, 2)); + renameSync(tmp, target); +} + +/** + * Returns null when the state file is absent, unreadable, or corrupt; callers + * treat that as "no recorded progress" and rerun the case. + */ +export function readState(caseDir: string): CaseState | null { + const path = statePath(caseDir); + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as CaseState; + } catch { + return null; + } +} + +/** True when `phase` is at or past `target`. infra_error is never at-target. */ +export function isAtLeast(phase: MusePhase, target: MusePhase): boolean { + if (phase === 'infra_error' || target === 'infra_error') return false; + return PHASE_ORDER[phase] >= PHASE_ORDER[target]; +} diff --git a/eval/lib/pool.test.ts b/eval/lib/pool.test.ts new file mode 100644 index 000000000..ea815c59a --- /dev/null +++ b/eval/lib/pool.test.ts @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { describe, expect, it } from 'vitest'; +import { mapPool } from './pool'; + +describe('mapPool', () => { + it('preserves result order', async () => { + const out = await mapPool([3, 1, 2], 2, async (n) => n * 10); + expect(out).toEqual([30, 10, 20]); + }); + + it('never exceeds the concurrency limit', async () => { + let active = 0; + let peak = 0; + await mapPool(Array.from({ length: 10 }, (_, i) => i), 3, async () => { + active++; + peak = Math.max(peak, active); + await new Promise((r) => setTimeout(r, 5)); + active--; + return true; + }); + expect(peak).toBeLessThanOrEqual(3); + expect(peak).toBeGreaterThan(1); + }); + + it('propagates a worker error', async () => { + await expect( + mapPool([1, 2], 2, async (n) => { + if (n === 1) throw new Error('boom'); + return n; + }), + ).rejects.toThrow('boom'); + }); + + it('handles an empty input', async () => { + expect(await mapPool([], 4, async () => 1)).toEqual([]); + }); + + it('rejects invalid limits', async () => { + await expect(mapPool([1], 0, async () => 1)).rejects.toThrow(/limit/); + await expect(mapPool([1], Number.NaN, async () => 1)).rejects.toThrow(/limit/); + }); + + it('passes the index and tolerates a limit larger than the input', async () => { + expect(await mapPool(['a', 'b'], 10, async (_item, i) => i)).toEqual([0, 1]); + }); +}); diff --git a/eval/lib/pool.ts b/eval/lib/pool.ts new file mode 100644 index 000000000..911361726 --- /dev/null +++ b/eval/lib/pool.ts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors + +/** + * Bounded-concurrency map. Results keep input order. Rejects with the first + * worker rejection; on rejection, in-flight and queued items are not cancelled + * and runners may keep processing until they finish. Callers are responsible + * for catching per-item errors when item isolation is required. + */ +export async function mapPool( + items: readonly T[], + limit: number, + worker: (item: T, index: number) => Promise, +): Promise { + if (!Number.isInteger(limit) || limit < 1) { + throw new Error(`mapPool: limit must be an integer >= 1, got ${limit}`); + } + const results = new Array(items.length); + let next = 0; + const runners = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await worker(items[index], index); + } + }); + await Promise.all(runners); + return results; +} diff --git a/eval/lib/systemPrompt.test.ts b/eval/lib/systemPrompt.test.ts new file mode 100644 index 000000000..082602625 --- /dev/null +++ b/eval/lib/systemPrompt.test.ts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { buildSystemPrompt, SWEEP_SKILLS } from './systemPrompt'; + +describe('buildSystemPrompt', () => { + it('joins selected skills in sorted order with separators', () => { + const root = mkdtempSync(join(tmpdir(), 'skills-')); + for (const name of ['b', 'a']) { + mkdirSync(join(root, name)); + writeFileSync(join(root, name, 'SKILL.md'), `# ${name}`); + } + const out = buildSystemPrompt(['b', 'a'], root); + expect(out).toBe('# a\n\n---\n\n# b'); + }); + + it('throws when a selected skill is missing', () => { + const root = mkdtempSync(join(tmpdir(), 'skills-')); + expect(() => buildSystemPrompt(['nope'], root)).toThrow("SKILL.md not found for skill 'nope'"); + }); + + it('ships the pilot skill selection by default', () => { + expect(SWEEP_SKILLS).toEqual([ + 'kernelcad', + 'kernelcad-authoring', + 'kernelcad-assemblies', + 'kernelcad-parts', + ]); + }); +}); diff --git a/eval/lib/systemPrompt.ts b/eval/lib/systemPrompt.ts new file mode 100644 index 000000000..b76630ddf --- /dev/null +++ b/eval/lib/systemPrompt.ts @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** Pilot-parity skill selection for MUSE/benchmark sweeps. */ +export const SWEEP_SKILLS = [ + 'kernelcad', + 'kernelcad-authoring', + 'kernelcad-assemblies', + 'kernelcad-parts', +] as const; + +export const SKILLS_ROOT = resolve('src/agent/skills'); + +/** Concatenate the SKILL.md files for the given skill dirs (sorted, safe). */ +export function buildSystemPrompt( + skillDirs: readonly string[], + root: string = SKILLS_ROOT, +): string { + const parts: string[] = []; + for (const name of [...skillDirs].sort()) { + const path = join(root, name, 'SKILL.md'); + if (!existsSync(path)) { + throw new Error(`SKILL.md not found for skill '${name}' at ${path}`); + } + parts.push(readFileSync(path, 'utf8')); + } + return parts.join('\n\n---\n\n'); +} diff --git a/eval/oracle/interference.ts b/eval/oracle/interference.ts index cdc810cf1..08bb8b0f0 100644 --- a/eval/oracle/interference.ts +++ b/eval/oracle/interference.ts @@ -13,6 +13,9 @@ import { existsSync } from 'node:fs'; const LOCAL_BUILD = './dist/cli/index.js'; +/** Hard cap on a single CLI invocation; runaway OCCT jobs are killed. */ +const CLI_TIMEOUT_MS = Number(process.env.KERNELCAD_CLI_TIMEOUT_MS ?? 300_000); + export interface InterferenceResult { ok: boolean; noSceneToCheck: boolean; @@ -36,13 +39,26 @@ function getBin(): { cmd: string; baseArgs: string[] } { async function runOnce(args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { const { cmd, baseArgs } = getBin(); return await new Promise((resolve, reject) => { - const child = spawn(cmd, [...baseArgs, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + const child = spawn(cmd, [...baseArgs, ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + timeout: CLI_TIMEOUT_MS, + }); let stdout = ''; let stderr = ''; child.stdout.on('data', (d) => (stdout += d.toString())); child.stderr.on('data', (d) => (stderr += d.toString())); child.on('error', reject); - child.on('close', (code) => resolve({ code: code ?? -1, stdout, stderr })); + child.on('close', (code, signal) => { + if (signal !== null && child.killed) { + resolve({ + code: -1, + stdout, + stderr: `${stderr}\n[timeout] kernelcad CLI killed after ${CLI_TIMEOUT_MS}ms (${signal})`, + }); + return; + } + resolve({ code: code ?? -1, stdout, stderr }); + }); }); } diff --git a/eval/oracle/kernelcad-client.ts b/eval/oracle/kernelcad-client.ts index 55fc165e8..cc79bc290 100644 --- a/eval/oracle/kernelcad-client.ts +++ b/eval/oracle/kernelcad-client.ts @@ -6,6 +6,9 @@ import type { EvaluateResult, ShapeInfo } from '../types'; const LOCAL_BUILD = './dist/cli/index.js'; +/** Hard cap on a single CLI invocation; runaway OCCT jobs are killed. */ +const CLI_TIMEOUT_MS = Number(process.env.KERNELCAD_CLI_TIMEOUT_MS ?? 300_000); + function getBin(): { cmd: string; baseArgs: string[] } { const override = process.env.KERNELCAD_BIN; if (override) { @@ -27,13 +30,26 @@ function getBin(): { cmd: string; baseArgs: string[] } { async function runOnce(args: string[], stdin?: string): Promise<{ code: number; stdout: string; stderr: string }> { const { cmd, baseArgs } = getBin(); return await new Promise((resolve, reject) => { - const child = spawn(cmd, [...baseArgs, ...args], { stdio: ['pipe', 'pipe', 'pipe'] }); + const child = spawn(cmd, [...baseArgs, ...args], { + stdio: ['pipe', 'pipe', 'pipe'], + timeout: CLI_TIMEOUT_MS, + }); let stdout = ''; let stderr = ''; child.stdout.on('data', (d) => (stdout += d.toString())); child.stderr.on('data', (d) => (stderr += d.toString())); child.on('error', reject); - child.on('close', (code) => resolve({ code: code ?? -1, stdout, stderr })); + child.on('close', (code, signal) => { + if (signal !== null && child.killed) { + resolve({ + code: -1, + stdout, + stderr: `${stderr}\n[timeout] kernelcad CLI killed after ${CLI_TIMEOUT_MS}ms (${signal})`, + }); + return; + } + resolve({ code: code ?? -1, stdout, stderr }); + }); if (stdin !== undefined) { child.stdin.write(stdin); } diff --git a/eval/oracle/museJudgeWrapper.py b/eval/oracle/museJudgeWrapper.py new file mode 100644 index 000000000..4ff87f1e6 --- /dev/null +++ b/eval/oracle/museJudgeWrapper.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Score one kernelCAD sample with MUSE's own alignment judge. + +Calls MUSE's published functions directly (`_run_alignment_judge`, +`_load_score_system_prompt`) with MUSE's judge model and temperature. +Writes the parsed result to --out as JSON. + +Exit code is always 0; transport errors are reported as {"error": ...}. +""" +import argparse +import json +import os +import sys +from pathlib import Path + +CATEGORY_KEYS = { + "assembly readiness": "assembly_readiness", + "joint design": "joint_design", + "tolerance": "tolerance", + "functional adaptation": "functional_adaptation", + "usage stability": "usage_stability", + "manufacturability": "manufacturability", +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--muse-root", required=True) + parser.add_argument("--case-name", required=True) + parser.add_argument("--case-dir", required=True) + parser.add_argument("--candidate-png", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--model", default="google/gemini-3.1-pro") + parser.add_argument("--base-url", default="https://api.deepinfra.com/v1/openai") + parser.add_argument("--api-key-env", default="DEEPINFRA_API_KEY") + parser.add_argument("--timeout", type=int, default=180) + args = parser.parse_args() + + api_key = os.environ.get(args.api_key_env) + if not api_key: + print(json.dumps({"error": f"missing env {args.api_key_env}"})) + return 0 + + muse_root = Path(args.muse_root).resolve() + sys.path.insert(0, str(muse_root / "src")) + try: + from judge_system.reverse_pipeline import ( # type: ignore + _load_score_system_prompt, + _run_alignment_judge, + ) + except Exception as exc: # pragma: no cover - env misconfiguration + print(json.dumps({"error": f"cannot import MUSE judge from {muse_root}: {exc}"})) + return 0 + + case_dir = Path(args.case_dir).resolve() + render_only = set() + render_list = muse_root / "src" / "judge_system" / "render_only_cases.txt" + if render_list.exists(): + for line in render_list.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + render_only.add(line) + if args.case_name in render_only: + reference = case_dir / f"{args.case_name}_stp_render.png" + else: + reference = case_dir / f"{args.case_name}.png" + + candidate = Path(args.candidate_png).resolve() + if not candidate.exists(): + print(json.dumps({"error": f"candidate render missing: {candidate}"})) + return 0 + if not reference.exists(): + print(json.dumps({"error": f"reference image missing: {reference}"})) + return 0 + + try: + payload = _run_alignment_judge( + api_key=api_key, + base_url=args.base_url, + model=args.model, + timeout_seconds=args.timeout, + system_prompt=_load_score_system_prompt(), + task_text=(case_dir / "design_description.md").read_text(encoding="utf-8"), + rubric_text=(case_dir / "evaluation_rubric.md").read_text(encoding="utf-8"), + candidate_svg_png=candidate, + reference_png=reference, + ) + except Exception as exc: + print(json.dumps({"error": f"judge call failed: {exc}"})) + return 0 + + categories = {} + for item in payload.get("items", []): + key = CATEGORY_KEYS.get(str(item.get("category_en", "")).strip().lower()) + if key: + try: + categories[key] = 1.0 if float(item.get("score", 0) or 0) >= 0.5 else 0.0 + except (TypeError, ValueError): + categories[key] = 0.0 + + overall = payload.get("overall_score_normalized") + if overall is None: + overall = payload.get("overall_score") + try: + overall_value = float(overall or 0.0) + if overall_value > 1: + overall_value = overall_value / 100.0 + except (TypeError, ValueError): + overall_value = 0.0 + + out = { + "overall": overall_value, + "categories": categories, + "summary": str(payload.get("overall_summary", "") or ""), + "items": payload.get("items", []), + "judge_model": args.model, + "judge_base_url": args.base_url, + "candidate_png": str(candidate), + "reference_png": str(reference), + } + Path(args.out).write_text(json.dumps(out, indent=2), encoding="utf-8") + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/oracle/museScorer.ts b/eval/oracle/museScorer.ts index 859b182db..d64da9ae4 100644 --- a/eval/oracle/museScorer.ts +++ b/eval/oracle/museScorer.ts @@ -33,6 +33,9 @@ import { fileURLToPath } from 'node:url'; const LOCAL_BUILD = './dist/cli/index.js'; const DEFAULT_MUSE_ROOT = '/home/andrii/projects/muse'; + +/** Hard cap on a single spawn; runaway OCCT/STEP jobs are killed. */ +const CLI_TIMEOUT_MS = Number(process.env.KERNELCAD_CLI_TIMEOUT_MS ?? 300_000); const __dirname = dirname(fileURLToPath(import.meta.url)); const WRAPPER_PY = resolve(__dirname, 'museScorerWrapper.py'); @@ -98,13 +101,26 @@ function getKernelcadBin(): { cmd: string; baseArgs: string[] } { function runOnce(cmd: string, args: string[]): Promise { return new Promise((resolveP, rejectP) => { - const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const child = spawn(cmd, args, { + stdio: ['ignore', 'pipe', 'pipe'], + timeout: CLI_TIMEOUT_MS, + }); let stdout = ''; let stderr = ''; child.stdout.on('data', (d) => (stdout += d.toString())); child.stderr.on('data', (d) => (stderr += d.toString())); child.on('error', rejectP); - child.on('close', (code) => resolveP({ code: code ?? -1, stdout, stderr })); + child.on('close', (code, signal) => { + if (signal !== null && child.killed) { + resolveP({ + code: -1, + stdout, + stderr: `${stderr}\n[timeout] process killed after ${CLI_TIMEOUT_MS}ms (${signal})`, + }); + return; + } + resolveP({ code: code ?? -1, stdout, stderr }); + }); }); } diff --git a/eval/runner.split.test.ts b/eval/runner.split.test.ts new file mode 100644 index 000000000..ce69ea453 --- /dev/null +++ b/eval/runner.split.test.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { generateCase, scoreCase } from './runner'; +import { MockAgentClient } from './agent'; +import type { AgentResponse } from './types'; + +const TASK_DIR = join(__dirname, 'tasks', 'bracket-holes'); +const SKILLS = '# skills\n\nBe precise.'; +const hasCli = + existsSync(join(process.cwd(), 'dist/cli/index.js')) || Boolean(process.env.KERNELCAD_BIN); + +const GOOD_SCRIPT = [ + '```ts', + 'export const bracket = box(40, 20, 5);', + '```', +].join('\n'); + +describe.skipIf(!hasCli)('generateCase / scoreCase split', () => { + it('generates without scoring, then scores the written script', async () => { + const runDir = mkdtempSync(join(tmpdir(), 'split-')); + const agent = new MockAgentClient([ + { text: GOOD_SCRIPT, tokens_in: 10, tokens_out: 5 } satisfies AgentResponse, + ]); + const gen = await generateCase({ + taskDir: TASK_DIR, + runDir, + agent, + model: 'mock-model', + skillMd: SKILLS, + startedAt: '2026-09-19T00-00-00', + candidates: 1, + maxAttempts: 3, + maxTokens: 8000, + temperature: 0.2, + }); + const script = readFileSync(gen.outputScriptPath, 'utf8'); + expect(script.length).toBeGreaterThan(0); + expect(gen.tokensIn).toBe(10); + + const result = await scoreCase({ + taskDir: TASK_DIR, + runDir, + outputScriptPath: gen.outputScriptPath, + events: gen.events, + attempts: gen.attempts, + tokensIn: gen.tokensIn, + tokensOut: gen.tokensOut, + generationMs: gen.timeMs, + startedAt: '2026-09-19T00-00-00', + model: 'mock-model', + firstFailureCode: gen.firstFailureCode, + noScript: gen.status === 'no_script', + }); + expect(result.task).toBe('bracket-holes'); + expect(result.score).not.toBeNull(); + expect(readFileSync(join(runDir, 'score.json'), 'utf8')).toContain('"attempts"'); + }); +}); diff --git a/eval/runner.ts b/eval/runner.ts index 261691a16..640035b46 100644 --- a/eval/runner.ts +++ b/eval/runner.ts @@ -2,7 +2,7 @@ // Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { join, resolve } from 'node:path'; -import type { AgentClient, TranscriptEvent, TaskResult, HarnessResult } from './types'; +import type { AgentClient, TranscriptEvent, TaskResult, HarnessResult, EvaluateResult } from './types'; import { extractScript, computeScore, renderTranscript } from './lib'; import { evaluateScript } from './oracle/kernelcad-client'; import { runClosedLoop, type LoopMessage } from '../src/agent/loop/closedLoop.js'; @@ -51,61 +51,74 @@ export interface RunTaskArgs { * deterministic. */ candidates?: number; + maxAttempts?: number; + maxTokens?: number; + /** Sent on every call when candidates <= 1 (sweep protocol temperature). */ + temperature?: number; } -export async function runTask(args: RunTaskArgs): Promise { +export interface GenerateCaseArgs { + taskDir: string; + runDir: string; + agent: AgentClient; + model: string; + skillMd: string; + startedAt: string; + cookbook?: CookbookInjection; + candidates?: number; + maxAttempts?: number; + maxTokens?: number; + /** Sent on every call when candidates <= 1 (sweep protocol temperature). */ + temperature?: number; +} + +export interface GenerateCaseResult { + events: TranscriptEvent[]; + status: 'passed' | 'gate_failed' | 'no_script'; + attempts: number; + tokensIn: number; + tokensOut: number; + timeMs: number; + firstFailureCode?: string; + outputScriptPath: string; +} + +export async function generateCase(args: GenerateCaseArgs): Promise { const taskDirAbs = resolve(args.taskDir); - const taskName = taskDirAbs.split('/').pop() ?? 'unknown'; - const promptPath = join(taskDirAbs, 'prompt.md'); - const harnessPath = join(taskDirAbs, 'harness.ts'); - const prompt = readFileSync(promptPath, 'utf8'); + const prompt = readFileSync(join(taskDirAbs, 'prompt.md'), 'utf8'); mkdirSync(args.runDir, { recursive: true }); const outputScriptPath = join(args.runDir, 'output.kcad.ts'); - const transcriptPath = join(args.runDir, 'transcript.md'); - const scorePath = join(args.runDir, 'score.json'); const events: TranscriptEvent[] = []; events.push({ kind: 'system_prompt', chars: args.skillMd.length }); events.push({ kind: 'user_prompt', content: prompt }); - if (args.cookbook) { - events.push({ - kind: 'cookbook_inject', - query: args.cookbook.query, - hits: args.cookbook.hits, - }); + events.push({ kind: 'cookbook_inject', query: args.cookbook.query, hits: args.cookbook.hits }); } - // Per-turn bookkeeping. `attemptNo` mirrors the closed loop's attempt index - // so the existing `'turn'`/`'evaluate'` transcript events keep their numbers. let attemptNo = 0; let totalIn = 0; let totalOut = 0; - // First non-OK diagnostic code observed across the loop. Set once, never - // overwritten — downstream classifiers (portfolio attempt logger) use this - // to tag a failed run with the diagnostic that surfaced first. let firstFailureCode: string | undefined; - const start = Date.now(); - // Drive the generate→gate→repair loop through the shared closed loop. The - // web gate runner gates on evaluate AND interference, so the loop now retries - // on interference failures too — not just on evaluate failures. + const candidates = args.candidates ?? 1; + const maxTokens = args.maxTokens ?? MAX_TOKENS; + const loopResult = await runClosedLoop({ prompt, gateRunner: createWebGateRunner(), extractScript, buildRepairPrompt, - maxAttempts: MAX_ATTEMPTS, - candidates: args.candidates ?? 1, + maxAttempts: args.maxAttempts ?? MAX_ATTEMPTS, + candidates, scoreCandidate: async (scriptPath, report) => { - // Only score build-valid candidates; gate-failing ones are ranked by stages. if (!report.ok) return null; try { const ev = await evaluateScript(scriptPath); if (!ev.ok) return null; - const harnessModule = await import(harnessPath); + const harnessModule = await import(join(taskDirAbs, 'harness.ts')); const hr = await harnessModule.default(scriptPath); return reduceHarnessScore(hr); } catch { @@ -119,13 +132,15 @@ export async function runTask(args: RunTaskArgs): Promise { generate: async (messages: LoopMessage[], opts?: { variant?: number }) => { attemptNo += 1; const turnStart = Date.now(); + const temperature = + candidates > 1 ? variantTemperature(opts?.variant) : args.temperature; const resp = await args.agent.generate({ system: args.skillMd, systemAddendum: args.cookbook?.systemPromptAddendum, messages: messages.map((m) => ({ role: m.role, content: m.content })), model: args.model, - max_tokens: MAX_TOKENS, - temperature: variantTemperature(opts?.variant), + max_tokens: maxTokens, + temperature, }); totalIn += resp.tokens_in; totalOut += resp.tokens_out; @@ -161,38 +176,61 @@ export async function runTask(args: RunTaskArgs): Promise { }, }); - // If we never extracted a script, write a placeholder so the human can read - // the run; the harness will be skipped and gate-fail recorded below. if (loopResult.status === 'no_script') { writeFileSync(outputScriptPath, '// (no script extracted from any attempt)'); } - // Re-run evaluate on the final written script to preserve the EXACT prior - // clean-decision + firstFailureCode semantics: the harness must still run - // when the script evaluates clean, even though the loop may have stopped on - // interference (which the harness does not itself gate on). - const finalEvaluate = - loopResult.status === 'no_script' - ? { - ok: false, - diagnostics: [ - { code: 'eval.no-script-extracted', message: 'No script extracted from any attempt.' }, - ], - } - : await evaluateScript(outputScriptPath); + return { + events, + status: loopResult.status, + attempts: loopResult.attempts, + tokensIn: totalIn, + tokensOut: totalOut, + timeMs: Date.now() - start, + firstFailureCode, + outputScriptPath, + }; +} + +export interface ScoreCaseArgs { + taskDir: string; + runDir: string; + outputScriptPath: string; + events?: TranscriptEvent[]; + attempts: number; + tokensIn: number; + tokensOut: number; + generationMs: number; + startedAt: string; + model: string; + firstFailureCode?: string; + /** True when generation never extracted a script — skips the evaluate call. */ + noScript?: boolean; +} + +export async function scoreCase(args: ScoreCaseArgs): Promise { + const taskDirAbs = resolve(args.taskDir); + const taskName = taskDirAbs.split('/').pop() ?? 'unknown'; + const events = args.events ?? []; + const scoringStart = Date.now(); + + const finalEvaluate: EvaluateResult = args.noScript + ? { + ok: false, + diagnostics: [ + { code: 'eval.no-script-extracted', message: 'No script extracted from any attempt.' }, + ], + } + : await evaluateScript(args.outputScriptPath); + let firstFailureCode = args.firstFailureCode; if (firstFailureCode === undefined && !finalEvaluate.ok && finalEvaluate.diagnostics.length > 0) { firstFailureCode = finalEvaluate.diagnostics[0].code; } - const lastEvaluateOk = finalEvaluate.ok; - // Run the task's harness against the final output. let harnessResult: HarnessResult; - if (lastEvaluateOk) { - const harnessModule = await import(harnessPath); - // Harnesses take an optional ctx so external scorers (cadqueryeval, MUSE) - // know where the task's reference artifacts live and where to write - // intermediates. Single-arg harnesses ignore the extra argument. - harnessResult = await harnessModule.default(outputScriptPath, { + if (finalEvaluate.ok) { + const harnessModule = await import(join(taskDirAbs, 'harness.ts')); + harnessResult = await harnessModule.default(args.outputScriptPath, { taskDir: taskDirAbs, runDir: args.runDir, }); @@ -207,16 +245,16 @@ export async function runTask(args: RunTaskArgs): Promise { }); const score = computeScore(harnessResult, { - attempts: loopResult.attempts, - tokens_in: totalIn, - tokens_out: totalOut, - time_ms: Date.now() - start, + attempts: args.attempts, + tokens_in: args.tokensIn, + tokens_out: args.tokensOut, + time_ms: args.generationMs + (Date.now() - scoringStart), firstFailureCode, }); - writeFileSync(scorePath, JSON.stringify(score, null, 2)); + writeFileSync(join(args.runDir, 'score.json'), JSON.stringify(score, null, 2)); writeFileSync( - transcriptPath, + join(args.runDir, 'transcript.md'), renderTranscript({ task: taskName, model: args.model, @@ -228,3 +266,33 @@ export async function runTask(args: RunTaskArgs): Promise { return { task: taskName, score }; } + +export async function runTask(args: RunTaskArgs): Promise { + const gen = await generateCase({ + taskDir: args.taskDir, + runDir: args.runDir, + agent: args.agent, + model: args.model, + skillMd: args.skillMd, + startedAt: args.startedAt, + cookbook: args.cookbook, + candidates: args.candidates, + maxAttempts: args.maxAttempts, + maxTokens: args.maxTokens, + temperature: args.temperature, + }); + return scoreCase({ + taskDir: args.taskDir, + runDir: args.runDir, + outputScriptPath: gen.outputScriptPath, + events: gen.events, + attempts: gen.attempts, + tokensIn: gen.tokensIn, + tokensOut: gen.tokensOut, + generationMs: gen.timeMs, + startedAt: args.startedAt, + model: args.model, + firstFailureCode: gen.firstFailureCode, + noScript: gen.status === 'no_script', + }); +} diff --git a/eval/runs/golden-muse-sweep/fixture.json b/eval/runs/golden-muse-sweep/fixture.json new file mode 100644 index 000000000..f971494a8 --- /dev/null +++ b/eval/runs/golden-muse-sweep/fixture.json @@ -0,0 +1,24 @@ +{ + "responses": [ + { + "text": "```ts\n// SPDX-License-Identifier: MIT\n// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors\n// MUSE 'stool' — four-legged CNC-milled timber stool, interlocking joinery.\n// 5 independent components: seat panel + 4 legs. Legs carry a square tenon\n// on top; the seat panel carries matching sockets cut into its bottom face\n// with assembly clearance (socket > tenon) so the assembled model is\n// interference-free while staying a tight interference-style fit in spirit.\n\n// Spec defaults (mm). seat_half_size raised to 250 within its documented\n// range so the assembled envelope matches the stated 500 x 500 x 415 overall.\nconst seatCornerRadius = 10;\nconst legCornerRadius = 3;\n\nconst LH = 400, SHS = 250, ST = 15, LHS = 20;\n// Leg centers: inset from the seat edge by one leg half-size + 10 mm margin.\nconst inset = LHS + 10;\nconst cx = SHS - inset; // 220 mm from center\n\nconst stool = assembly('stool');\n\n// --- Seat panel: rounded-rect plate with four bottom sockets -------------\n// extrudeRoundedRect is center-anchored in XY, base at z=0.\nconst seatPlate = extrudeRoundedRect(2 * SHS, 2 * SHS, seatCornerRadius, ST)\n .translate(0, 0, LH);\nconst socketCenters: [number, number][] = [\n [cx, cx], [-cx, cx], [cx, -cx], [-cx, -cx],\n];\nlet seatShape = seatPlate;\nfor (const [sx, sy] of socketCenters) {\n // Socket: rectangular pocket cut upward into the bottom face.\n const socket = box(2 * 5.2, 2 * 5.2, 7.7, true).translate(sx, sy, LH + 7.7 / 2 - 0.01);\n seatShape = seatShape.subtract(socket);\n}\nconst seat = stool.part('seat_panel', seatShape.color('plate'));\n\n// --- Legs: rounded-square column + top tenon ------------------------------\nfunction makeLeg(lx: number, ly: number) {\n const column = extrudeRoundedRect(2 * LHS, 2 * LHS, legCornerRadius, LH); // center-anchored\n // Tenon stops 0.2 mm short of the socket floor (socket 7.7 vs tenon 7.5).\n const tenon = box(2 * 5, 2 * 5, 7.5, true).translate(0, 0, LH + 7.5 / 2);\n return column.union(tenon).translate(lx, ly, 0).color('beam');\n}\n\nconst legNames = [\n ['front_left_leg', -cx, -cx],\n ['front_right_leg', cx, -cx],\n ['rear_left_leg', -cx, cx],\n ['rear_right_leg', cx, cx],\n] as const;\n\nfor (const [name, lx, ly] of legNames) {\n const leg = stool.part(name, makeLeg(lx, ly));\n // Connector pair at the tenon/socket interface (seat-bottom plane).\n leg.connector('tenon-top', { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n seat.connector(`socket-${name}`, { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n stool.mate(`${name}-to-seat`, `seat_panel.socket-${name}`, `${name}.tenon-top`, 'fastened');\n}\n\nreturn stool.solvedModel({});\n\n```", + "tokens_in": 1, + "tokens_out": 1 + }, + { + "text": "```ts\n// SPDX-License-Identifier: MIT\n// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors\n// MUSE 'stool' — four-legged CNC-milled timber stool, interlocking joinery.\n// 5 independent components: seat panel + 4 legs. Legs carry a square tenon\n// on top; the seat panel carries matching sockets cut into its bottom face\n// with assembly clearance (socket > tenon) so the assembled model is\n// interference-free while staying a tight interference-style fit in spirit.\n\n// Spec defaults (mm). seat_half_size raised to 250 within its documented\n// range so the assembled envelope matches the stated 500 x 500 x 415 overall.\nconst seatCornerRadius = 10;\nconst legCornerRadius = 3;\n\nconst LH = 400, SHS = 250, ST = 15, LHS = 20;\n// Leg centers: inset from the seat edge by one leg half-size + 10 mm margin.\nconst inset = LHS + 10;\nconst cx = SHS - inset; // 220 mm from center\n\nconst stool = assembly('stool');\n\n// --- Seat panel: rounded-rect plate with four bottom sockets -------------\n// extrudeRoundedRect is center-anchored in XY, base at z=0.\nconst seatPlate = extrudeRoundedRect(2 * SHS, 2 * SHS, seatCornerRadius, ST)\n .translate(0, 0, LH);\nconst socketCenters: [number, number][] = [\n [cx, cx], [-cx, cx], [cx, -cx], [-cx, -cx],\n];\nlet seatShape = seatPlate;\nfor (const [sx, sy] of socketCenters) {\n // Socket: rectangular pocket cut upward into the bottom face.\n const socket = box(2 * 5.2, 2 * 5.2, 7.7, true).translate(sx, sy, LH + 7.7 / 2 - 0.01);\n seatShape = seatShape.subtract(socket);\n}\nconst seat = stool.part('seat_panel', seatShape.color('plate'));\n\n// --- Legs: rounded-square column + top tenon ------------------------------\nfunction makeLeg(lx: number, ly: number) {\n const column = extrudeRoundedRect(2 * LHS, 2 * LHS, legCornerRadius, LH); // center-anchored\n // Tenon stops 0.2 mm short of the socket floor (socket 7.7 vs tenon 7.5).\n const tenon = box(2 * 5, 2 * 5, 7.5, true).translate(0, 0, LH + 7.5 / 2);\n return column.union(tenon).translate(lx, ly, 0).color('beam');\n}\n\nconst legNames = [\n ['front_left_leg', -cx, -cx],\n ['front_right_leg', cx, -cx],\n ['rear_left_leg', -cx, cx],\n ['rear_right_leg', cx, cx],\n] as const;\n\nfor (const [name, lx, ly] of legNames) {\n const leg = stool.part(name, makeLeg(lx, ly));\n // Connector pair at the tenon/socket interface (seat-bottom plane).\n leg.connector('tenon-top', { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n seat.connector(`socket-${name}`, { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n stool.mate(`${name}-to-seat`, `seat_panel.socket-${name}`, `${name}.tenon-top`, 'fastened');\n}\n\nreturn stool.solvedModel({});\n\n```", + "tokens_in": 1, + "tokens_out": 1 + }, + { + "text": "```ts\n// SPDX-License-Identifier: MIT\n// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors\n// MUSE 'stool' — four-legged CNC-milled timber stool, interlocking joinery.\n// 5 independent components: seat panel + 4 legs. Legs carry a square tenon\n// on top; the seat panel carries matching sockets cut into its bottom face\n// with assembly clearance (socket > tenon) so the assembled model is\n// interference-free while staying a tight interference-style fit in spirit.\n\n// Spec defaults (mm). seat_half_size raised to 250 within its documented\n// range so the assembled envelope matches the stated 500 x 500 x 415 overall.\nconst seatCornerRadius = 10;\nconst legCornerRadius = 3;\n\nconst LH = 400, SHS = 250, ST = 15, LHS = 20;\n// Leg centers: inset from the seat edge by one leg half-size + 10 mm margin.\nconst inset = LHS + 10;\nconst cx = SHS - inset; // 220 mm from center\n\nconst stool = assembly('stool');\n\n// --- Seat panel: rounded-rect plate with four bottom sockets -------------\n// extrudeRoundedRect is center-anchored in XY, base at z=0.\nconst seatPlate = extrudeRoundedRect(2 * SHS, 2 * SHS, seatCornerRadius, ST)\n .translate(0, 0, LH);\nconst socketCenters: [number, number][] = [\n [cx, cx], [-cx, cx], [cx, -cx], [-cx, -cx],\n];\nlet seatShape = seatPlate;\nfor (const [sx, sy] of socketCenters) {\n // Socket: rectangular pocket cut upward into the bottom face.\n const socket = box(2 * 5.2, 2 * 5.2, 7.7, true).translate(sx, sy, LH + 7.7 / 2 - 0.01);\n seatShape = seatShape.subtract(socket);\n}\nconst seat = stool.part('seat_panel', seatShape.color('plate'));\n\n// --- Legs: rounded-square column + top tenon ------------------------------\nfunction makeLeg(lx: number, ly: number) {\n const column = extrudeRoundedRect(2 * LHS, 2 * LHS, legCornerRadius, LH); // center-anchored\n // Tenon stops 0.2 mm short of the socket floor (socket 7.7 vs tenon 7.5).\n const tenon = box(2 * 5, 2 * 5, 7.5, true).translate(0, 0, LH + 7.5 / 2);\n return column.union(tenon).translate(lx, ly, 0).color('beam');\n}\n\nconst legNames = [\n ['front_left_leg', -cx, -cx],\n ['front_right_leg', cx, -cx],\n ['rear_left_leg', -cx, cx],\n ['rear_right_leg', cx, cx],\n] as const;\n\nfor (const [name, lx, ly] of legNames) {\n const leg = stool.part(name, makeLeg(lx, ly));\n // Connector pair at the tenon/socket interface (seat-bottom plane).\n leg.connector('tenon-top', { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n seat.connector(`socket-${name}`, { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n stool.mate(`${name}-to-seat`, `seat_panel.socket-${name}`, `${name}.tenon-top`, 'fastened');\n}\n\nreturn stool.solvedModel({});\n\n```", + "tokens_in": 1, + "tokens_out": 1 + }, + { + "text": "```ts\n// SPDX-License-Identifier: MIT\n// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors\n// MUSE 'stool' — four-legged CNC-milled timber stool, interlocking joinery.\n// 5 independent components: seat panel + 4 legs. Legs carry a square tenon\n// on top; the seat panel carries matching sockets cut into its bottom face\n// with assembly clearance (socket > tenon) so the assembled model is\n// interference-free while staying a tight interference-style fit in spirit.\n\n// Spec defaults (mm). seat_half_size raised to 250 within its documented\n// range so the assembled envelope matches the stated 500 x 500 x 415 overall.\nconst seatCornerRadius = 10;\nconst legCornerRadius = 3;\n\nconst LH = 400, SHS = 250, ST = 15, LHS = 20;\n// Leg centers: inset from the seat edge by one leg half-size + 10 mm margin.\nconst inset = LHS + 10;\nconst cx = SHS - inset; // 220 mm from center\n\nconst stool = assembly('stool');\n\n// --- Seat panel: rounded-rect plate with four bottom sockets -------------\n// extrudeRoundedRect is center-anchored in XY, base at z=0.\nconst seatPlate = extrudeRoundedRect(2 * SHS, 2 * SHS, seatCornerRadius, ST)\n .translate(0, 0, LH);\nconst socketCenters: [number, number][] = [\n [cx, cx], [-cx, cx], [cx, -cx], [-cx, -cx],\n];\nlet seatShape = seatPlate;\nfor (const [sx, sy] of socketCenters) {\n // Socket: rectangular pocket cut upward into the bottom face.\n const socket = box(2 * 5.2, 2 * 5.2, 7.7, true).translate(sx, sy, LH + 7.7 / 2 - 0.01);\n seatShape = seatShape.subtract(socket);\n}\nconst seat = stool.part('seat_panel', seatShape.color('plate'));\n\n// --- Legs: rounded-square column + top tenon ------------------------------\nfunction makeLeg(lx: number, ly: number) {\n const column = extrudeRoundedRect(2 * LHS, 2 * LHS, legCornerRadius, LH); // center-anchored\n // Tenon stops 0.2 mm short of the socket floor (socket 7.7 vs tenon 7.5).\n const tenon = box(2 * 5, 2 * 5, 7.5, true).translate(0, 0, LH + 7.5 / 2);\n return column.union(tenon).translate(lx, ly, 0).color('beam');\n}\n\nconst legNames = [\n ['front_left_leg', -cx, -cx],\n ['front_right_leg', cx, -cx],\n ['rear_left_leg', -cx, cx],\n ['rear_right_leg', cx, cx],\n] as const;\n\nfor (const [name, lx, ly] of legNames) {\n const leg = stool.part(name, makeLeg(lx, ly));\n // Connector pair at the tenon/socket interface (seat-bottom plane).\n leg.connector('tenon-top', { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n seat.connector(`socket-${name}`, { type: 'frame', origin: { kind: 'vec3', value: [lx, ly, LH] }, axis: [0, 0, 1] });\n stool.mate(`${name}-to-seat`, `seat_panel.socket-${name}`, `${name}.tenon-top`, 'fastened');\n}\n\nreturn stool.solvedModel({});\n\n```", + "tokens_in": 1, + "tokens_out": 1 + } + ] +} \ No newline at end of file diff --git a/eval/tasks/muse-bookshelf/harness.ts b/eval/tasks/muse-bookshelf/harness.ts new file mode 100644 index 000000000..8fd0b4012 --- /dev/null +++ b/eval/tasks/muse-bookshelf/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-bookshelf/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'bookshelf' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-bookshelf'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-bookshelf/prompt.md b/eval/tasks/muse-bookshelf/prompt.md new file mode 100644 index 000000000..ae2e23094 --- /dev/null +++ b/eval/tasks/muse-bookshelf/prompt.md @@ -0,0 +1,118 @@ +# bookshelf (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a multi-tier wooden bookshelf designed for load-bearing storage, utilizing a modular board-and-slat architecture with dowel-based assembly. + +## Geometry and Dimensions +Approx. 600.0 mm × 280.0 mm × 1200.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Load-bearing storage for books and household items. + +## Structural Features +Four vertical uprights; horizontal side rails (bottom, top, and intermediate tiers); horizontal shelf slats. + +## Special Requirements +Keep assembly split unchanged. Ensure all dowel hole alignments remain strictly coaxial between mating parts. + +## Planned Component Quantity +44 + +## Component Names +- right_front_upright +- right_back_upright +- left_front_upright +- left_back_upright +- right_bottom_rail +- left_bottom_rail +- right_top_rail +- left_top_rail +- tier_1_left_rail +- tier_1_right_rail +- tier_1_slat_01 to tier_1_slat_07 +- tier_2_left_rail +- tier_2_right_rail +- tier_2_slat_01 to tier_2_slat_07 +- tier_3_left_rail +- tier_3_right_rail +- tier_3_slat_01 to tier_3_slat_07 +- tier_4_left_rail +- tier_4_right_rail +- tier_4_slat_01 to tier_4_slat_07 + +## Adjustable Parameters +- **shelf_width**: 600.0 (400.0 ~ 900.0 mm). Determines the overall width of the bookshelf and the span of the horizontal slats. +- **shelf_depth**: 280.0 (200.0 ~ 400.0 mm). Controls the footprint depth and determines the maximum number of slats per tier. +- **shelf_height**: 1200.0 (800.0 ~ 1800.0 mm). Sets the total vertical height and influences the spacing between tiers. +- **num_tiers**: 4 (2.0 ~ 6.0). Defines the number of intermediate storage levels (shelves) excluding the top and bottom structural rails. +- **board_thickness**: 10.0 (6.0 ~ 16.0 mm). Defines the material thickness for all uprights, rails, and slats, ensuring adequate structural rigidity. +- **board_width**: 30.0 (20.0 ~ 45.0 mm). Determines the width of the structural framing members and individual shelf slats. +- **slat_gap**: 7.0 (3.0 ~ 12.0 mm). Controls the spacing between adjacent slats on a tier to optimize material usage and aesthetics. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Sets the radius of the blind holes used for the dowel pins. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth of the dowel pins into the timber boards to ensure joint stability. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~4. Uprights (Right Front, Right Back, Left Front, Left Back) +The primary vertical supports of the bookshelf. +* **Component Purpose**: Vertical structural support. Transfers the load of the shelves and stored items to the ground. +* **Assembly Direction**: Vertical base components, positioned along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features blind holes on the inner Y-faces to receive the horizontal rails at the bottom, top, and all intermediate tier levels. + +### 5~8. Bottom and Top Rails (Right & Left) +The outer horizontal framing members. +* **Component Purpose**: Structural framing. Connects the front and back uprights at the extreme top and bottom to prevent racking and ensure frame rigidity. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features dowel holes on the -Y and +Y end faces mating with the uprights. + +### 9~16. Tier Rails (Left & Right for Tiers 1 to 4) +The intermediate horizontal supports for the shelves. +* **Component Purpose**: Load-bearing supports that connect the uprights at specific heights and provide a resting base and alignment interface for the shelf slats. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features holes on the end faces for upright connection, and upward-facing holes on the +Z face to align and secure the slats. + +### 17~44. Shelf Slats (Tiers 1 to 4, Slats 01 to 07) +The horizontal surfaces of the bookshelf. +* **Component Purpose**: Forms the distributed load-bearing surface of the shelves for storing items. +* **Assembly Direction**: Placed downwards along the -Z axis onto the tier rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features blind holes on the bottom (-Z) face at each end to align with the corresponding holes on the tier rails. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 44-component model: + +* **Bottom Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to the inner faces of the front and back uprights at the base. +* **Top Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to the inner faces of the front and back uprights at the top. +* **Tier Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to the inner faces of the front and back uprights at evenly distributed Z-heights. +* **Shelf Slats -> Tier Rails** | Joint: Dowel Joint | Note: Slat bottom faces connect to the top faces of the left and right tier rails via dowel pins. diff --git a/eval/tasks/muse-chair3_stool_2/harness.ts b/eval/tasks/muse-chair3_stool_2/harness.ts new file mode 100644 index 000000000..25fc913cb --- /dev/null +++ b/eval/tasks/muse-chair3_stool_2/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair3_stool_2/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair3_stool_2' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair3_stool_2'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair3_stool_2/prompt.md b/eval/tasks/muse-chair3_stool_2/prompt.md new file mode 100644 index 000000000..67b1a819b --- /dev/null +++ b/eval/tasks/muse-chair3_stool_2/prompt.md @@ -0,0 +1,112 @@ +# chair3_stool_2 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged bar stool with staggered horizontal footrests designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 350.0 mm × 350.0 mm × 780.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person elevated seating (bar or counter use). + +## Structural Features +Seat panel; four legs; four horizontal footrests (front, rear, left, right). + +## Special Requirements +Keep assembly split unchanged. Ensure staggered Z-heights for X-axis and Y-axis footrests to prevent internal tenon collision within the legs. + +## Planned Component Quantity +9 + +## Component Names +- Seat panel +- Front left leg +- Front right leg +- Rear left leg +- Rear right leg +- Front footrest +- Rear footrest +- Left footrest +- Right footrest + +## Adjustable Parameters +- **width**: 350 (250.0 ~ 500.0 mm). Defines the overall width of the stool base and seat. +- **depth**: 350 (250.0 ~ 500.0 mm). Defines the overall depth of the stool base and seat. +- **seat_height**: 750 (600.0 ~ 900.0 mm). Determines the height of the seating surface, suitable for bar or counter ergonomics. +- **leg_thickness**: 35 (20.0 ~ 60.0 mm). Ensures structural stability and load-bearing capacity of the vertical supports. +- **seat_thickness**: 30 (15.0 ~ 50.0 mm). Provides sufficient material depth for the leg tenons to insert securely. +- **tenon_length**: 20 (8.0 ~ 40.0 mm). Controls the insertion depth of the joints for mechanical strength. +- **tenon_offset**: 5 (2.0 ~ 15.0 mm). Defines the setback of the tenon to prevent edge splitting. +- **footrest_height**: 300 (150.0 ~ 500.0 mm). Sets the ergonomic height for resting feet and provides lower structural bracing. +- **footrest_thickness**: 22 (12.0 ~ 40.0 mm). Determines the robustness of the horizontal bracing. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base for seating and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = `seat_height` + `seat_thickness` / 2.0. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four rectangular sockets to receive the leg tenons. + +### 2~5. Four Legs (Front Left, Front Right, Rear Left, Rear Right) +The vertical supporting entities of the stool. +* **Component Purpose**: Transfers the seat load to the ground, ensuring anti-overturning stability. Houses mortise sockets for the footrests. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of length `tenon_length` that interference-fits into the bottom sockets of the seat panel. Inner faces feature sockets at staggered heights for the footrests. + +### 6~7. Front and Rear Footrests +The X-axis horizontal bracing entities. +* **Component Purpose**: Connects the left and right legs to prevent splay, enhances structural rigidity, and serves as a footrest. +* **Assembly Direction**: Inserted horizontally along the X axis into the corresponding legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the leg sockets at the base `footrest_height`. + +### 8~9. Left and Right Footrests +The Y-axis horizontal bracing entities. +* **Component Purpose**: Connects the front and rear legs to prevent splay, enhances structural rigidity, and serves as a footrest. +* **Assembly Direction**: Inserted horizontally along the Y axis into the corresponding legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the leg sockets at `footrest_height` + `footrest_thickness` to avoid internal collision with the front/rear footrest tenons. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **Front Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-left socket. +* **Front Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-right socket. +* **Rear Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-left socket. +* **Rear Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-right socket. +* **Front Footrest -> Front Left & Right Legs** | Joint: interlocking | Note: Tenons inserted into inner X-facing sockets of the front legs. +* **Rear Footrest -> Rear Left & Right Legs** | Joint: interlocking | Note: Tenons inserted into inner X-facing sockets of the rear legs. +* **Left Footrest -> Front & Rear Left Legs** | Joint: interlocking | Note: Tenons inserted into inner Y-facing sockets of the left legs (staggered Z-height). +* **Right Footrest -> Front & Rear Right Legs** | Joint: interlocking | Note: Tenons inserted into inner Y-facing sockets of the right legs (staggered Z-height). +* **Seat Panel -> All Legs** | Joint: Support Base | Note: Acts as the core hub; all vertical connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-chair_2/harness.ts b/eval/tasks/muse-chair_2/harness.ts new file mode 100644 index 000000000..37db9a7bc --- /dev/null +++ b/eval/tasks/muse-chair_2/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_2/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_2' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_2'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_2/prompt.md b/eval/tasks/muse-chair_2/prompt.md new file mode 100644 index 000000000..d6252c22e --- /dev/null +++ b/eval/tasks/muse-chair_2/prompt.md @@ -0,0 +1,81 @@ +# chair_2 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a two-piece minimalist chair consisting of a lower base frame and an upper seat-and-backrest frame, assembled via a central interlocking joint. + +## Geometry and Dimensions +Approx. 420.0 mm × 613.0 mm × 898.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Lower base frame; upper seat and backrest frame. + +## Special Requirements +Keep assembly split unchanged. Maintain the defined installation clearance for the joint to ensure proper physical assembly. + +## Planned Component Quantity +2 + +## Component Names +- lower_frame_with_tenon +- upper_seat_back_frame + +## Adjustable Parameters +- **seat_width**: 420.0 (320.0 ~ 520.0 mm). Determines the overall width of the chair and the extrusion depth of the 2D profiles, directly affecting seating area and stability. +- **joint_width**: 20.0 (10.0 ~ 40.0 mm). Controls the thickness of the connecting tenon, balancing structural shear strength and material limits. +- **joint_end_clearance**: 40.0 (15.0 ~ 90.0 mm). Defines the setback distance of the tenon from the lateral edges to prevent material breakout or splitting at the joint ends. +- **profile_fillet_radius**: 4.0 (1.0 ~ 10.0 mm). Sets the corner rounding of the profile, providing ergonomic safety and accommodating CNC tool radius compensation. +- **joint_clearance**: 0.2 (0.0 ~ 1.0 mm). Provides the necessary dimensional tolerance for the physical assembly of the mortise and tenon joint. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. lower_frame_with_tenon +The supporting base entity of the chair. +* **Component Purpose**: Acts as the main load-bearing base, transferring weight to the ground and providing the male tenon interface for the upper frame. +* **Assembly Direction**: Fixed base component, positioned as the foundational structure. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a top-centered protruding rectangular tenon. + +### 2. upper_seat_back_frame +The functional support entity of the chair. +* **Component Purpose**: Provides the seating surface and backrest for human-computer interaction, featuring a central slot to mate securely with the lower frame. +* **Assembly Direction**: Inserted downwards onto the lower frame's top tenon. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a central mortise (slot cut) at the mating face that accommodates the lower frame's tenon with a defined clearance. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **upper_seat_back_frame -> lower_frame_with_tenon** | Joint: interlocking | Note: Upper frame's central slot fits over the lower frame's top tenon. diff --git a/eval/tasks/muse-chair_2_rocker/harness.ts b/eval/tasks/muse-chair_2_rocker/harness.ts new file mode 100644 index 000000000..2abae7db0 --- /dev/null +++ b/eval/tasks/muse-chair_2_rocker/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_2_rocker/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_2_rocker' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_2_rocker'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_2_rocker/prompt.md b/eval/tasks/muse-chair_2_rocker/prompt.md new file mode 100644 index 000000000..fdd213224 --- /dev/null +++ b/eval/tasks/muse-chair_2_rocker/prompt.md @@ -0,0 +1,81 @@ +# chair_2_rocker (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a two-piece rocking chair consisting of a lower rocker frame and an upper seat/backrest, designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 674.0 mm × 460.0 mm × 872.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating (rocking chair). + +## Structural Features +Lower rocker frame; upper seat and backrest. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +2 + +## Component Names +- lower_frame +- upper_seat_back + +## Adjustable Parameters +- **seat_width**: 460 (360.0 ~ 560.0 mm). Determines the seating width and overall extrusion depth of the chair. +- **joint_width**: 20 (10.0 ~ 40.0 mm). Controls the thickness of the connecting tenon, balancing structural strength and available material. +- **joint_end_clearance**: 40 (15.0 ~ 90.0 mm). Defines the setback of the joint from the edges of the seat to prevent wood splitting at the ends. +- **profile_fillet_radius**: 4 (1.0 ~ 10.0 mm). Smooths the sharp corners of the side profiles for ergonomics, aesthetics, and machining tool radius constraints. +- **joint_clearance**: 0.2 (0.0 ~ 1.0 mm). Provides tolerance for the mortise and tenon fit to ensure smooth assemblability without being too loose. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. lower_frame +The supporting base of the rocking chair. +* **Component Purpose**: Acts as the rocking base and lower support structure, transferring the load to the ground while enabling the rocking motion. +* **Assembly Direction**: Fixed base component. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a protruding rectangular tenon at the top center to connect with the upper seat. + +### 2. upper_seat_back +The functional support entity of the chair. +* **Component Purpose**: Provides the seating surface and backrest for human-computer interaction and ergonomic support. +* **Assembly Direction**: Placed downwards along the -Z axis onto the lower frame. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features a central slot (mortise) at the bottom to receive the lower frame's tenon. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **upper_seat_back -> lower_frame** | Joint: interlocking | Note: Upper seat's bottom slot receives the lower frame's top tenon. diff --git a/eval/tasks/muse-chair_2_stool/harness.ts b/eval/tasks/muse-chair_2_stool/harness.ts new file mode 100644 index 000000000..d7d265277 --- /dev/null +++ b/eval/tasks/muse-chair_2_stool/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_2_stool/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_2_stool' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_2_stool'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_2_stool/prompt.md b/eval/tasks/muse-chair_2_stool/prompt.md new file mode 100644 index 000000000..1d1eb8e46 --- /dev/null +++ b/eval/tasks/muse-chair_2_stool/prompt.md @@ -0,0 +1,81 @@ +# chair_2_stool (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a two-piece stool designed for wood-based assembly, featuring a lower support frame and an upper seat cap. + +## Geometry and Dimensions +Approx. 490.0 mm × 400.0 mm × 400.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Lower frame; seat cap. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +2 + +## Component Names +- lower_frame +- seat_cap + +## Adjustable Parameters +- **seat_width**: 400.0 (300.0 ~ 500.0 mm). Determines the seating depth and overall extrusion length of the stool. +- **joint_width**: 20.0 (10.0 ~ 40.0 mm). Controls the thickness of the tenon, balancing structural strength and available material. +- **joint_end_clearance**: 40.0 (15.0 ~ 90.0 mm). Defines the setback distance of the tenon from the part edges to prevent wood splitting at the ends. +- **profile_fillet_radius**: 4.0 (1.0 ~ 10.0 mm). Smooths the sharp corners of the side profiles for ergonomics, safety, and aesthetics. +- **joint_clearance**: 0.2 (0.0 ~ 1.0 mm). Provides manufacturing tolerance for the mortise and tenon fit to ensure smooth assemblability. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. lower_frame +The supporting base of the stool. +* **Component Purpose**: Acts as the main load-bearing structure, transferring the user's weight to the ground while providing a physical connection interface for the seat cap. +* **Assembly Direction**: Fixed base component, positioned at absolute ground level. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The top center features a protruding rectangular tenon. + +### 2. seat_cap +The functional seating entity of the stool. +* **Component Purpose**: Provides the horizontal seating surface for human interaction and locks onto the lower frame to complete the structure. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the lower frame. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom features a dedicated slot (mortise) that receives the tenon of the lower frame. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **seat_cap -> lower_frame** | Joint: interlocking | Note: Seat cap bottom slot receives the lower frame's top tenon. diff --git a/eval/tasks/muse-chair_3/harness.ts b/eval/tasks/muse-chair_3/harness.ts new file mode 100644 index 000000000..af48a3ab6 --- /dev/null +++ b/eval/tasks/muse-chair_3/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3/prompt.md b/eval/tasks/muse-chair_3/prompt.md new file mode 100644 index 000000000..cd05a27bb --- /dev/null +++ b/eval/tasks/muse-chair_3/prompt.md @@ -0,0 +1,124 @@ +# chair_3 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a slatted wooden chair with a backrest, designed for modular dowel-based assembly. + +## Geometry and Dimensions +Approx. 400.0 mm × 400.0 mm × 609.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat slats; under-seat rails; vertical uprights (front and rear legs); horizontal side rails; back slats. + +## Special Requirements +Keep assembly split unchanged. All components must retain their respective alignment holes for dowel insertion. + +## Planned Component Quantity +24 + +## Component Names +- seat_slat_01 to seat_slat_11 (11 components) +- left_under_seat_rail +- right_under_seat_rail +- right_back_upright +- right_front_upright +- right_lower_side_rail +- right_upper_side_rail +- left_back_upright +- left_front_upright +- left_lower_side_rail +- left_upper_side_rail +- back_slat_01 to back_slat_03 (3 components) + +## Adjustable Parameters +- **board_length**: 609.0 (450.0 ~ 760.0 mm). Determines the overall height of the back uprights and the chair's maximum vertical dimension. +- **board_thickness**: 10.0 (6.0 ~ 24.0 mm). Controls the structural thickness of all wooden members; lower limits may compromise load-bearing capacity. +- **seat_length**: 400.0 (320.0 ~ 520.0 mm). Defines the depth of the seating area, affecting ergonomic comfort. +- **seat_height**: 400.0 (320.0 ~ 520.0 mm). Sets the distance from the floor to the seating surface, adhering to standard seating postures. +- **board_width**: 30.0 (15.0 ~ 60.0 mm). Determines the width of the slats and structural rails, balancing weight and stiffness. +- **seat_board_gap**: 6.96 (2.0 ~ 18.0 mm). Controls the spacing between individual seat slats for aesthetics and material economy. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Sets the radius of the cylindrical cuts used for the dowel joints. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth for the dowel pins to ensure adequate joint strength. +- **horizontal_bottom_board_length**: 400.0 (300.0 ~ 560.0 mm). Defines the overall width of the chair base and seating area. +- **back_slat_spacing**: 35.0 (20.0 ~ 60.0 mm). Controls the vertical distribution of the backrest slats. +- **back_slat_top_margin**: 25.2 (12.0 ~ 60.0 mm). Sets the clearance from the top of the back uprights to the highest back slat. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like dowel holes via boolean cuts. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~11. Seat Slats (seat_slat_01 to seat_slat_11) +The horizontal surfaces forming the seating area. +* **Component Purpose**: Directly supports the user's weight. Features through-holes for alignment and fastening to the under-seat rails. +* **Assembly Direction**: Placed horizontally along the Z-axis at `seat_height`. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Connected to the under-seat rails via vertical cylindrical holes. + +### 12~13. Under-Seat Rails (left_under_seat_rail, right_under_seat_rail) +The primary horizontal load-bearing supports for the seat. +* **Component Purpose**: Bridges the front and back uprights and provides a mounting base for the seat slats. +* **Assembly Direction**: Positioned longitudinally along the Y-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical holes on the top face for seat slats and horizontal holes on the ends for the uprights. + +### 14, 18. Back Uprights (right_back_upright, left_back_upright) +The rear vertical structural pillars. +* **Component Purpose**: Acts as the rear legs and extends upwards to support the back slats. +* **Assembly Direction**: Vertical placement along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features horizontal holes to receive the under-seat rails, side rails, and back slats. + +### 15, 19. Front Uprights (right_front_upright, left_front_upright) +The front vertical structural pillars. +* **Component Purpose**: Acts as the front legs supporting the front edge of the seat. +* **Assembly Direction**: Vertical placement along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features horizontal holes to receive the under-seat rails and side rails. + +### 16~17, 20~21. Side Rails (right_lower/upper, left_lower/upper) +Horizontal stabilizers connecting the front and rear legs. +* **Component Purpose**: Prevents splaying of the legs and increases the overall rigidity of the chair frame. +* **Assembly Direction**: Horizontal placement along the Y-axis at specific Z-heights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Ends feature holes aligning with the front and back uprights. + +### 22~24. Back Slats (back_slat_01 to back_slat_03) +The horizontal supports for the user's back. +* **Component Purpose**: Provides ergonomic lumbar and back support. +* **Assembly Direction**: Horizontal placement along the X-axis between the rear uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Ends feature holes aligning with the inner faces of the back uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 24-component model: + +* **Seat Slats (01-11) -> Under-Seat Rails (Left/Right)** | Joint: Dowel Joint | Note: Vertical dowels connect the bottom of the slats to the top of the rails. +* **Under-Seat Rails (Left/Right) -> Front & Back Uprights** | Joint: Dowel Joint | Note: Horizontal dowels connect the ends of the rails to the inner faces of the uprights. +* **Side Rails (Upper/Lower, Left/Right) -> Front & Back Uprights** | Joint: Dowel Joint | Note: Horizontal dowels connect the ends of the side rails to the uprights for lateral stability. +* **Back Slats (01-03) -> Back Uprights (Left/Right)** | Joint: Dowel Joint | Note: Horizontal dowels connect the ends of the back slats to the upper inner faces of the back uprights. diff --git a/eval/tasks/muse-chair_3_double_back/harness.ts b/eval/tasks/muse-chair_3_double_back/harness.ts new file mode 100644 index 000000000..840c0e1d8 --- /dev/null +++ b/eval/tasks/muse-chair_3_double_back/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3_double_back/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3_double_back' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3_double_back'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3_double_back/prompt.md b/eval/tasks/muse-chair_3_double_back/prompt.md new file mode 100644 index 000000000..cebb1a35f --- /dev/null +++ b/eval/tasks/muse-chair_3_double_back/prompt.md @@ -0,0 +1,118 @@ +# chair_3_double_back (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a slatted wooden chair featuring a double-sided frame, a multi-slat backrest, and a slatted seat, designed for dowel-based assembly. + +## Geometry and Dimensions +Approx. 425.0 mm × 420.0 mm × 750.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Dowel Joint + +## Mechanical Condition +Single-person seating. + +## Structural Features +Slatted seat panel; left and right side frames (each with front/back uprights and upper/lower rails); under-seat support rails; multi-slat backrest. + +## Special Requirements +Keep assembly split unchanged. Ensure all cylindrical cuts for dowel pins align perfectly between mating components. + +## Planned Component Quantity +28 + +## Component Names +- seat_slat_01 to seat_slat_12 +- left_under_seat_rail +- right_under_seat_rail +- right_back_upright +- right_front_upright +- right_lower_side_rail +- right_upper_side_rail +- left_back_upright +- left_front_upright +- left_lower_side_rail +- left_upper_side_rail +- back_slat_01 to back_slat_06 + +## Adjustable Parameters +- **board_length**: 750.0 (600.0 ~ 900.0 mm). Determines the overall height of the chair and the backrest uprights. +- **board_thickness**: 10.0 (6.0 ~ 24.0 mm). Controls the structural thickness of the slats and frame members. +- **seat_length**: 400.0 (320.0 ~ 520.0 mm). Determines the transverse width of the seating area. +- **seat_height**: 400.0 (320.0 ~ 520.0 mm). Ergonomic height of the seat surface from the ground. +- **board_width**: 25.0 (12.0 ~ 50.0 mm). Width of the individual slats and structural frame members. +- **seat_board_gap**: 6.96 (2.0 ~ 18.0 mm). Spacing between adjacent seat slats to allow for visual design and material expansion. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Radius of the cylindrical cuts used for the dowel joints. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Depth of the blind holes for the dowel insertions. +- **horizontal_bottom_board_length**: 400.0 (300.0 ~ 560.0 mm). Determines the depth of the chair's side frames and under-seat rails. +- **back_slat_spacing**: 28.0 (18.0 ~ 50.0 mm). Vertical center-to-center distance between the backrest slats. +- **back_slat_top_margin**: 20.0 (10.0 ~ 45.0 mm). Distance from the top of the back uprights to the highest back slat. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~12. Seat Slats (`seat_slat_01` to `seat_slat_12`) +The horizontal surfaces forming the seating area. +* **Component Purpose**: Directly supports the user's weight. Distributes the load to the under-seat rails. +* **Assembly Direction**: Placed downwards along the -Z axis onto the under-seat rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features cylindrical holes on the bottom face to align with the under-seat rails. + +### 13~14. Under Seat Rails (`left_under_seat_rail`, `right_under_seat_rail`) +The primary horizontal load-bearing beams beneath the seat. +* **Component Purpose**: Supports the seat slats and transfers the vertical load to the front and back uprights. +* **Assembly Direction**: Horizontal placement along the Y axis between the uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical holes on the top edge for seat slats and horizontal holes on the ends for the uprights. + +### 15, 16, 19, 20. Side Uprights (`right_back_upright`, `right_front_upright`, `left_back_upright`, `left_front_upright`) +The main vertical structural pillars of the chair. +* **Component Purpose**: Transfers all loads to the ground. The back uprights extend upwards to support the back slats. +* **Assembly Direction**: Vertical standing components, acting as the base frame reference. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features multiple blind holes on the inner faces to receive side rails, under-seat rails, and back slats. + +### 17, 18, 21, 22. Side Rails (`right_lower_side_rail`, `right_upper_side_rail`, `left_lower_side_rail`, `left_upper_side_rail`) +Horizontal braces for the left and right side frames. +* **Component Purpose**: Prevents the front and back uprights from splaying, ensuring structural rigidity of the side frames. +* **Assembly Direction**: Inserted horizontally along the Y axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features holes on the end faces mating with the uprights. + +### 23~28. Back Slats (`back_slat_01` to `back_slat_06`) +The horizontal boards forming the backrest. +* **Component Purpose**: Provides lumbar and back support for the user. Ties the left and right back uprights together. +* **Assembly Direction**: Inserted horizontally along the X axis between the left and right back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features holes on the end faces mating with the inner faces of the back uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 28-component model: + +* **Seat Slats (01-12) -> Under Seat Rails** | Joint: Dowel Joint | Note: Bottom of seat slats connect to the top edge of the under-seat rails. +* **Under Seat Rails -> Front & Back Uprights** | Joint: Dowel Joint | Note: Ends of the under-seat rails connect to the inner faces of the uprights. +* **Lower/Upper Side Rails -> Front & Back Uprights** | Joint: Dowel Joint | Note: Ends of the side rails connect to the inner faces of the uprights to form rigid side frames. +* **Back Slats (01-06) -> Left & Right Back Uprights** | Joint: Dowel Joint | Note: Ends of the back slats connect to the inner faces of the extended back uprights. diff --git a/eval/tasks/muse-chair_3_kids/harness.ts b/eval/tasks/muse-chair_3_kids/harness.ts new file mode 100644 index 000000000..5095da3a5 --- /dev/null +++ b/eval/tasks/muse-chair_3_kids/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3_kids/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3_kids' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3_kids'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3_kids/prompt.md b/eval/tasks/muse-chair_3_kids/prompt.md new file mode 100644 index 000000000..67671fa64 --- /dev/null +++ b/eval/tasks/muse-chair_3_kids/prompt.md @@ -0,0 +1,118 @@ +# chair_3_kids (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a child-sized chair composed of wooden slats and boards, featuring a slatted seat and backrest designed for dowel-based assembly. + +## Geometry and Dimensions +Approx. 322.0 mm × 316.0 mm × 440.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Dowel Joint + +## Mechanical Condition +Single-child seating, load-bearing furniture. + +## Structural Features +Slatted seat panel; under-seat support rails; four vertical uprights (legs/backrest supports); horizontal side rails; slatted backrest. + +## Special Requirements +Keep assembly split unchanged. Ensure all dowel holes align perfectly between mating components. + +## Planned Component Quantity +24 + +## Component Names +- seat_slat_01 ~ seat_slat_11 +- left_under_seat_rail +- right_under_seat_rail +- right_back_upright +- right_front_upright +- right_lower_side_rail +- right_upper_side_rail +- left_back_upright +- left_front_upright +- left_lower_side_rail +- left_upper_side_rail +- back_slat_01 ~ back_slat_03 + +## Adjustable Parameters +- **board_length**: 440.0 (350.0 ~ 550.0 mm). Determines the overall height of the chair and the backrest uprights. +- **board_thickness**: 8.0 (5.0 ~ 16.0 mm). Affects the structural robustness and weight of the chair components. +- **seat_length**: 300.0 (240.0 ~ 400.0 mm). Determines the depth of the seating area. +- **seat_height**: 280.0 (220.0 ~ 350.0 mm). Strictly follows ergonomic standards for a child's seating posture. +- **board_width**: 22.0 (12.0 ~ 40.0 mm). Width of the individual slats and frame members. +- **seat_board_gap**: 5.0 (2.0 ~ 12.0 mm). Controls the spacing between seat slats for aesthetics and material savings. +- **hole_radius**: 0.8 (0.4 ~ 2.0 mm). Radius of the dowel holes for assembly connections. +- **hole_depth**: 2.5 (1.0 ~ 6.0 mm). Determines the insertion depth for the dowel pins. +- **horizontal_bottom_board_length**: 300.0 (240.0 ~ 400.0 mm). Length of the lower side rails providing base stability. +- **back_slat_spacing**: 28.0 (16.0 ~ 50.0 mm). Vertical gap between backrest slats. +- **back_slat_top_margin**: 20.0 (10.0 ~ 40.0 mm). Distance from the top of the uprights to the first back slat. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like dowel holes via boolean cuts. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~11. Seat Slats (seat_slat_01 to seat_slat_11) +The primary seating surface of the chair. +* **Component Purpose**: Provides direct load-bearing support for the user. +* **Assembly Direction**: Placed horizontally along the X-axis, spaced evenly along the Y-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Bottom faces feature cylindrical blind holes that mate with the under-seat rails. + +### 12~13. Under Seat Rails (left_under_seat_rail, right_under_seat_rail) +The primary horizontal supports for the seat. +* **Component Purpose**: Bridges the front and back uprights and provides a mounting base for the seat slats. +* **Assembly Direction**: Horizontal placement along the Y-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Top faces feature holes for seat slats; side faces feature holes for connecting to the vertical uprights. + +### 14, 15, 18, 19. Vertical Uprights (right_back, right_front, left_back, left_front) +The main vertical structural pillars. +* **Component Purpose**: Transfers all loads to the ground. The back uprights extend upwards to support the back slats. +* **Assembly Direction**: Vertical insertion along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Inner faces feature blind holes to receive the horizontal side rails, under-seat rails, and back slats. + +### 16, 17, 20, 21. Side Horizontal Rails (right_lower, right_upper, left_lower, left_upper) +Lateral stabilizers for the chair frame. +* **Component Purpose**: Connects the front and rear uprights at the top and bottom to prevent racking and ensure structural rigidity. +* **Assembly Direction**: Horizontal placement along the Y-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Ends feature dowel holes mating with the vertical uprights. + +### 22~24. Back Slats (back_slat_01 to back_slat_03) +The functional back support entity. +* **Component Purpose**: Provides lumbar and back support for the child. +* **Assembly Direction**: Horizontal placement along the X-axis, stacked vertically. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Ends feature dowel holes mating with the inner faces of the left and right back uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 24-component model: + +* **Seat Slats -> Under Seat Rails** | Joint: Dowel Joint | Note: Bottom of seat slats pinned to the top of the under-seat rails. +* **Under Seat Rails -> Vertical Uprights** | Joint: Dowel Joint | Note: Ends of under-seat rails pinned to the inner faces of the front and back uprights. +* **Side Horizontal Rails -> Vertical Uprights** | Joint: Dowel Joint | Note: Ends of horizontal rails pinned to the front and back uprights at upper and lower positions. +* **Back Slats -> Back Uprights** | Joint: Dowel Joint | Note: Ends of back slats pinned to the inner faces of the left and right back uprights. diff --git a/eval/tasks/muse-chair_3_stool/harness.ts b/eval/tasks/muse-chair_3_stool/harness.ts new file mode 100644 index 000000000..e3f658b51 --- /dev/null +++ b/eval/tasks/muse-chair_3_stool/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3_stool/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3_stool' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3_stool'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3_stool/prompt.md b/eval/tasks/muse-chair_3_stool/prompt.md new file mode 100644 index 000000000..bdb2c2619 --- /dev/null +++ b/eval/tasks/muse-chair_3_stool/prompt.md @@ -0,0 +1,110 @@ +# chair_3_stool (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a backless slatted stool designed for board-based assembly using dowel pins. + +## Geometry and Dimensions +Approx. 410.0 mm × 400.0 mm × 450.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Dowel Joint + +## Mechanical Condition +Single-person seating. + +## Structural Features +Slatted seat surface; left side support frame; right side support frame; under-seat support rails. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +20 + +## Component Names +- seat_slat_01 through seat_slat_10 +- left_under_seat_rail +- right_under_seat_rail +- right_front_upright +- right_rear_upright +- right_lower_side_rail +- right_upper_side_rail +- left_front_upright +- left_rear_upright +- left_lower_side_rail +- left_upper_side_rail + +## Adjustable Parameters +- **board_length**: 450.0 (360.0 ~ 560.0 mm). Determines the overall height of the stool uprights. +- **board_thickness**: 10.0 (6.0 ~ 24.0 mm). Defines the thickness of the structural boards, ensuring adequate material for dowel hole depth. +- **board_width**: 30.0 (15.0 ~ 60.0 mm). Defines the width of the frame members and seat slats. +- **seat_length**: 380.0 (300.0 ~ 480.0 mm). Determines the depth of the seating area. +- **seat_height**: 420.0 (360.0 ~ 520.0 mm). Strictly follows ergonomic standards for seating posture. +- **seat_board_gap**: 6.96 (2.0 ~ 18.0 mm). Controls the spacing between the seat slats, affecting the total number of slats generated. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Defines the radius of the cylindrical cuts for the dowel pins. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth for the dowel connections. +- **horizontal_bottom_board_length**: 380.0 (300.0 ~ 480.0 mm). Defines the length of the horizontal side rails connecting the uprights. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like cylindrical holes for dowel joints. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~10. Seat Slats (seat_slat_01 to seat_slat_10) +The horizontal seating surface of the stool. +* **Component Purpose**: Acts as the primary load-bearing surface for the user, distributing weight to the under-seat rails. +* **Assembly Direction**: Placed horizontally along the X-axis, stacked along the Y-axis at absolute $Z = seat\_height$. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Bottom faces feature cylindrical holes to align with the under-seat rails. + +### 11~12. Under Seat Rails (Left & Right) +The primary horizontal supports beneath the seat. +* **Component Purpose**: Bridges the front and rear uprights while providing a mounting base for the seat slats. +* **Assembly Direction**: Positioned horizontally along the Y-axis beneath the seat slats. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Top face features holes for the seat slats; side faces feature holes to connect to the uprights. + +### 13, 14, 17, 18. Uprights (Right Front, Right Rear, Left Front, Left Rear) +The vertical supporting legs of the stool. +* **Component Purpose**: Transfers the load from the seat to the ground and acts as the vertical framework for the side rails. +* **Assembly Direction**: Positioned vertically along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Inner faces feature cylindrical holes to receive the horizontal side rails and under-seat rails. + +### 15, 16, 19, 20. Side Rails (Right Lower, Right Upper, Left Lower, Left Upper) +The horizontal bracing for the side frames. +* **Component Purpose**: Connects the front and rear uprights on each side to prevent racking and ensure structural stability. +* **Assembly Direction**: Positioned horizontally along the Y-axis between the uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). End faces feature cylindrical holes aligning with the uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 20-component model: + +* **Seat Slats -> Under Seat Rails** | Joint: Dowel Joint | Note: Bottom of slats align with top of under-seat rails via dowel pins. +* **Left Under Seat Rail -> Left Uprights** | Joint: Dowel Joint | Note: Ends of the rail connect to the inner faces of the left front and rear uprights. +* **Right Under Seat Rail -> Right Uprights** | Joint: Dowel Joint | Note: Ends of the rail connect to the inner faces of the right front and rear uprights. +* **Left Side Rails (Upper/Lower) -> Left Uprights** | Joint: Dowel Joint | Note: Horizontal rails bridge the left front and rear uprights. +* **Right Side Rails (Upper/Lower) -> Right Uprights** | Joint: Dowel Joint | Note: Horizontal rails bridge the right front and rear uprights. diff --git a/eval/tasks/muse-chair_3_tall/harness.ts b/eval/tasks/muse-chair_3_tall/harness.ts new file mode 100644 index 000000000..aa26df5c0 --- /dev/null +++ b/eval/tasks/muse-chair_3_tall/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3_tall/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3_tall' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3_tall'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3_tall/prompt.md b/eval/tasks/muse-chair_3_tall/prompt.md new file mode 100644 index 000000000..84180c689 --- /dev/null +++ b/eval/tasks/muse-chair_3_tall/prompt.md @@ -0,0 +1,121 @@ +# chair_3_tall (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a bar-height chair with a footrest tier and slatted seat and backrest, designed for modular board assembly. + +## Geometry and Dimensions +Approx. 410.0 mm × 404.0 mm × 900.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Dowel Joint + +## Mechanical Condition +Single-person seating at bar or counter height, requiring elevated foot support and structural stability. + +## Structural Features +Slatted seat panel; four vertical uprights (legs); under-seat support rails; horizontal side rails (upper and lower); footrest rails; slatted backrest. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +25 + +## Component Names +- seat_slat_01 to seat_slat_10 +- left_under_seat_rail +- right_under_seat_rail +- right_back_upright +- right_front_upright +- right_lower_side_rail +- right_upper_side_rail +- right_footrest_rail +- left_back_upright +- left_front_upright +- left_lower_side_rail +- left_upper_side_rail +- left_footrest_rail +- back_slat_01 to back_slat_03 + +## Adjustable Parameters +- **board_length**: 900.0 (750.0 ~ 1050.0 mm). Determines the overall height of the chair and the backrest. +- **board_thickness**: 12.0 (8.0 ~ 24.0 mm). Controls the thickness of all structural boards, affecting weight and load-bearing capacity. +- **seat_length**: 380.0 (300.0 ~ 480.0 mm). Defines the depth of the seating area. +- **seat_height**: 650.0 (550.0 ~ 750.0 mm). Sets the ergonomic height for bar/counter seating. +- **board_width**: 30.0 (15.0 ~ 60.0 mm). Determines the width of the individual slats and frame members. +- **seat_board_gap**: 6.96 (2.0 ~ 18.0 mm). Controls the spacing between the seat slats for aesthetics and material savings. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Defines the radius of the cylindrical cuts used for the dowel/screw joints. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth for the connecting dowels/hardware. +- **horizontal_bottom_board_length**: 380.0 (280.0 ~ 500.0 mm). Defines the depth of the side frames and overall footprint stability. +- **back_slat_spacing**: 40.0 (25.0 ~ 65.0 mm). Controls the vertical gap between backrest slats. +- **back_slat_top_margin**: 28.0 (14.0 ~ 55.0 mm). Sets the clearance from the top of the back uprights to the first back slat. +- **footrest_height**: 250.0 (150.0 ~ 400.0 mm). Ergonomic placement of the footrest rail to support the user's legs. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~10. Seat Slats +The primary seating surface. +* **Component Purpose**: Provides the horizontal load-bearing surface for the user. +* **Assembly Direction**: Placed horizontally on top of the under-seat rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features cylindrical holes on the bottom face to align with the under-seat rails. + +### 11~12. Under Seat Rails (Left, Right) +The primary support for the seat slats. +* **Component Purpose**: Bridges the front and back uprights, providing a mounting base for the seat slats and transferring the user's weight to the legs. +* **Assembly Direction**: Positioned horizontally along the Y-axis, intersecting the vertical uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical holes for the seat slats and horizontal holes for connecting to the vertical uprights. + +### 13~14 & 18~19. Vertical Uprights (Right Back, Right Front, Left Back, Left Front) +The main vertical structural pillars (legs). +* **Component Purpose**: Transfers all loads to the ground and provides the framework for all horizontal rails and back slats. The back uprights extend upwards to support the backrest. +* **Assembly Direction**: Positioned vertically along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features multiple cylindrical holes along their length to accept side rails, under-seat rails, and back slats. + +### 15~17 & 20~22. Horizontal Side Rails (Lower, Upper, Footrest - Left & Right) +The lateral bracing elements. +* **Component Purpose**: Prevents the chair from splaying or wobbling. The footrest rails specifically provide ergonomic support for the user's feet. +* **Assembly Direction**: Positioned horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features cylindrical holes at both ends to mate with the vertical uprights. + +### 23~25. Back Slats +The lumbar and back support. +* **Component Purpose**: Provides a resting surface for the user's back, bridging the extended left and right back uprights. +* **Assembly Direction**: Positioned horizontally along the X-axis between the rear uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features cylindrical holes at the ends to connect to the inner faces of the back uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 25-component model: + +* **Seat Slats -> Under Seat Rails** | Joint: Dowel Joint | Note: Bottom of seat slats connect to the top edge of the under-seat rails. +* **Under Seat Rails -> Vertical Uprights** | Joint: Dowel Joint | Note: Ends of the under-seat rails connect to the inner faces of the front and back uprights. +* **Horizontal Side Rails -> Vertical Uprights** | Joint: Dowel Joint | Note: Ends of the upper, lower, and footrest rails connect to the inner faces of the front and back uprights. +* **Back Slats -> Back Vertical Uprights** | Joint: Dowel Joint | Note: Ends of the back slats connect to the upper inner faces of the left and right back uprights. diff --git a/eval/tasks/muse-chair_3_wide_back/harness.ts b/eval/tasks/muse-chair_3_wide_back/harness.ts new file mode 100644 index 000000000..6e04e81bb --- /dev/null +++ b/eval/tasks/muse-chair_3_wide_back/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_3_wide_back/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_3_wide_back' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_3_wide_back'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_3_wide_back/prompt.md b/eval/tasks/muse-chair_3_wide_back/prompt.md new file mode 100644 index 000000000..d172e7a89 --- /dev/null +++ b/eval/tasks/muse-chair_3_wide_back/prompt.md @@ -0,0 +1,128 @@ +# chair_3_wide_back (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a slatted-seat chair with a single wide back panel and side rail supports, designed for dowel-based wood assembly. + +## Geometry and Dimensions +Approx. 430.0 mm × 420.0 mm × 650.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Dowel Joint + +## Mechanical Condition +Single-person seating. + +## Structural Features +Multiple seat slats; two under-seat rails; four vertical uprights (legs); four horizontal side rails; single wide back panel. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +22 + +## Component Names +- seat_slat_01 +- seat_slat_02 +- seat_slat_03 +- seat_slat_04 +- seat_slat_05 +- seat_slat_06 +- seat_slat_07 +- seat_slat_08 +- seat_slat_09 +- seat_slat_10 +- seat_slat_11 +- left_under_seat_rail +- right_under_seat_rail +- right_back_upright +- right_front_upright +- right_lower_side_rail +- right_upper_side_rail +- left_back_upright +- left_front_upright +- left_lower_side_rail +- left_upper_side_rail +- back_panel + +## Adjustable Parameters +- **board_length**: 650.0 (500.0 ~ 800.0 mm). Determines the overall height of the chair and the vertical uprights. +- **board_thickness**: 10.0 (6.0 ~ 24.0 mm). Defines the thickness of the structural boards and panels. +- **seat_length**: 400.0 (320.0 ~ 520.0 mm). Determines the depth of the seating area. +- **seat_height**: 400.0 (320.0 ~ 520.0 mm). Sets the ergonomic height of the seating surface from the ground. +- **board_width**: 30.0 (15.0 ~ 60.0 mm). Width of the structural frame boards and seat slats. +- **seat_board_gap**: 6.96 (2.0 ~ 18.0 mm). Controls the spacing between individual seat slats. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Radius of the cylindrical cuts used for dowel connections. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth for the dowel joints. +- **horizontal_bottom_board_length**: 400.0 (300.0 ~ 560.0 mm). Length of the horizontal side rails connecting the front and back uprights. +- **back_panel_height**: 180.0 (100.0 ~ 300.0 mm). Defines the vertical coverage of the wide backrest panel. +- **back_panel_margin**: 20.0 (10.0 ~ 50.0 mm). Sets the vertical clearance between the seat surface and the bottom edge of the back panel. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~11. Seat Slats (seat_slat_01 to seat_slat_11) +The horizontal seating surface components. +* **Component Purpose**: Distributes the user's weight across the under-seat rails. +* **Assembly Direction**: Placed downwards along the -Z axis onto the under-seat rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical cylindrical holes on the bottom face to align with the under-seat rails. + +### 12~13. Under-Seat Rails (left_under_seat_rail, right_under_seat_rail) +The primary horizontal load-bearing supports for the seat. +* **Component Purpose**: Bridges the front and back uprights and provides a mounting base for the seat slats. +* **Assembly Direction**: Horizontal alignment along the Y-axis between the uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical holes on the top face for the slats, and horizontal holes on the ends to connect to the uprights. + +### 14, 15, 18, 19. Vertical Uprights (right_back, right_front, left_back, left_front) +The main vertical structural pillars (legs). +* **Component Purpose**: Transfers all loads to the ground and provides the framework for rails and the back panel. +* **Assembly Direction**: Vertical base components. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features multiple horizontal cylindrical holes to receive the under-seat rails, side rails, and back panel. + +### 16, 17, 20, 21. Side Rails (right_lower, right_upper, left_lower, left_upper) +Horizontal stabilizers connecting the front and back legs. +* **Component Purpose**: Prevents splaying of the legs and increases the overall rigidity of the chair frame. +* **Assembly Direction**: Horizontal insertion along the Y-axis into the uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features horizontal holes at both ends mating with the uprights. + +### 22. Back Panel +The single wide lumbar/back support. +* **Component Purpose**: Provides ergonomic back support for the user and adds lateral stability to the upper rear frame. +* **Assembly Direction**: Horizontal insertion along the X-axis between the left and right back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features horizontal holes on its side edges mating with the inner faces of the back uprights. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 22-component model: + +* **Seat Slats (01-11) -> Under-Seat Rails** | Joint: Dowel Joint | Note: Bottom of slats connect to top of under-seat rails. +* **Under-Seat Rails -> Front & Back Uprights** | Joint: Dowel Joint | Note: Ends of under-seat rails connect to the inner faces of the uprights. +* **Upper & Lower Side Rails -> Front & Back Uprights** | Joint: Dowel Joint | Note: Ends of side rails connect to the inner faces of the uprights at top and bottom positions. +* **Back Panel -> Left & Right Back Uprights** | Joint: Dowel Joint | Note: Sides of the back panel connect to the inner faces of the rear uprights. diff --git a/eval/tasks/muse-chair_4/harness.ts b/eval/tasks/muse-chair_4/harness.ts new file mode 100644 index 000000000..fc7d8cd86 --- /dev/null +++ b/eval/tasks/muse-chair_4/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_4/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_4' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_4'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_4/prompt.md b/eval/tasks/muse-chair_4/prompt.md new file mode 100644 index 000000000..761672bc0 --- /dev/null +++ b/eval/tasks/muse-chair_4/prompt.md @@ -0,0 +1,104 @@ +# chair_4 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a laminated profile chair formed by a series of parallel, vertically oriented wooden panels that create a continuous S-curve for the base, seat, and backrest. + +## Geometry and Dimensions +Approx. 560.0 mm × 493.9 mm × 1015.2 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Single-person seating. + +## Structural Features +28 parallel S-curve profile panels arranged in a mirrored configuration. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +28 + +## Component Names +- profile_panel_01 +- profile_panel_02 +- profile_panel_03 +- profile_panel_04 +- profile_panel_05 +- profile_panel_06 +- profile_panel_07 +- profile_panel_08 +- profile_panel_09 +- profile_panel_10 +- profile_panel_11 +- profile_panel_12 +- profile_panel_13 +- profile_panel_14 +- profile_panel_15 +- profile_panel_16 +- profile_panel_17 +- profile_panel_18 +- profile_panel_19 +- profile_panel_20 +- profile_panel_21 +- profile_panel_22 +- profile_panel_23 +- profile_panel_24 +- profile_panel_25 +- profile_panel_26 +- profile_panel_27 +- profile_panel_28 + +## Adjustable Parameters +- **overall_height**: 1015.2 (700.0 ~ 1300.0 mm). Controls the total height of the backrest, affecting lumbar and shoulder support. +- **panel_depth**: 20.0 (8.0 ~ 40.0 mm). Determines the thickness of each individual wooden slice, balancing structural rigidity and overall weight. +- **panel_count**: 28 (8 ~ 64). Dictates the resolution of the lamination and the density of the ribbed structure. +- **seat_width**: 560.0 (260.0 ~ 620.0 mm). Defines the total ergonomic seating width across the arrayed panels. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~28. Profile Panels 01 to 28 +The structural and ergonomic slices of the chair. +* **Component Purpose**: When arrayed together, these 2D-profiled panels form the continuous 3D surface of the chair, acting simultaneously as the floor base, the seating surface, and the backrest. +* **Assembly Direction**: Arrayed horizontally along the Y-axis, with the second half mirrored across the center plane. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 28-component model: + +* **profile_panel_01 -> profile_panel_02** | Joint: Bonding (Glue) | Note: Sequential face-to-face lamination along the Y-axis. +* **profile_panel_02 -> profile_panel_03** | Joint: Bonding (Glue) | Note: Sequential face-to-face lamination along the Y-axis. +* **profile_panel_03 -> profile_panel_04** | Joint: Bonding (Glue) | Note: Sequential face-to-face lamination along the Y-axis. +* **...** | Joint: Bonding (Glue) | Note: Pattern continues for all adjacent panels. +* **profile_panel_27 -> profile_panel_28** | Joint: Bonding (Glue) | Note: Final sequential face-to-face lamination along the Y-axis. diff --git a/eval/tasks/muse-chair_cross/harness.ts b/eval/tasks/muse-chair_cross/harness.ts new file mode 100644 index 000000000..76c3aa766 --- /dev/null +++ b/eval/tasks/muse-chair_cross/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_cross/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_cross' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_cross'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_cross/prompt.md b/eval/tasks/muse-chair_cross/prompt.md new file mode 100644 index 000000000..3416a1621 --- /dev/null +++ b/eval/tasks/muse-chair_cross/prompt.md @@ -0,0 +1,113 @@ +# chair_cross (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged dining chair with a backrest and cross stretchers designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 430.0 mm × 410.0 mm × 850.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four legs; backrest panel; front stretcher; rear stretcher. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +8 + +## Component Names +- Seat panel +- Front left leg +- Front right leg +- Rear left leg +- Rear right leg +- Backrest panel +- Front stretcher +- Rear stretcher + +## Adjustable Parameters +- **width**: 430 (300.0 ~ 700.0 mm). Constrains the extreme values to prevent tipping caused by unbalanced length-to-width ratios. +- **depth**: 410 (300.0 ~ 650.0 mm). Determines the seating depth for ergonomic comfort. +- **seat_height**: 450 (350.0 ~ 520.0 mm). Strictly follows ergonomic standards for single-person seating posture. +- **backrest_height**: 370 (200.0 ~ 550.0 mm). Provides adequate lumbar support without raising the center of gravity too high. +- **leg_thickness**: 40 (20.0 ~ 70.0 mm). Lower limit ensures load-bearing stiffness; upper limit prevents interference and material waste. +- **seat_thickness**: 30 (15.0 ~ 50.0 mm). Must be thick enough to accommodate the insertion depth of the leg and backrest tenons. +- **stretcher_height**: 130 (60.0 ~ 280.0 mm). Determines the vertical placement of the stretchers to optimize leg stability and prevent splaying. +- **stretcher_thickness**: 22 (12.0 ~ 45.0 mm). Ensures the stretchers are robust enough to handle horizontal tension and compression forces. +- **tenon_length**: 20 (8.0 ~ 15.0 mm). Determines the bite depth of the physical connections. *(Note: Default value exceeds the defined dictionary range, but dictates the physical insertion depth).* +- **tenon_offset**: 5 (2.0 ~ 20.0 mm). Controls the setback distance of the tenon relative to the part edge to prevent wood splitting. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the chair. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the legs and backrest. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = `seat_height`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four rectangular sockets for the legs; rear features a socket for the backrest. + +### 2~5. Four Legs (Front Left, Front Right, Rear Left, Rear Right) +The supporting entities of the chair. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability in the X-Y plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of length `tenon_length` that interference-fits into the bottom sockets of the seat panel. The inner X-facing side features a mortise to receive the stretcher tenons. + +### 6. Backrest Panel +The functional support entity of the chair. +* **Component Purpose**: Vertical guide. Provides back support for human-computer interaction, ensuring structural strength under large torque via a mortise-and-tenon joint. +* **Assembly Direction**: Pressed downwards along the -Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features a tenon inserted into the dedicated socket at the rear of the seat panel. + +### 7~8. Stretchers (Front, Rear) +The horizontal bracing entities of the chair. +* **Component Purpose**: Horizontal support. Connects the left and right legs to prevent splaying and significantly increases the overall structural rigidity of the base. +* **Assembly Direction**: Inserted horizontally along the X axis between the left and right legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the inner faces of the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 8-component model: + +* **Front Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-left socket. +* **Front Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-right socket. +* **Rear Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-left socket. +* **Rear Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-right socket. +* **Backrest Panel -> Seat Panel** | Joint: interlocking | Note: Backrest bottom tenon inserted into seat's rear socket. +* **Front Stretcher -> Front Left Leg** | Joint: interlocking | Note: Left tenon inserted into front-left leg's inner mortise. +* **Front Stretcher -> Front Right Leg** | Joint: interlocking | Note: Right tenon inserted into front-right leg's inner mortise. +* **Rear Stretcher -> Rear Left Leg** | Joint: interlocking | Note: Left tenon inserted into rear-left leg's inner mortise. +* **Rear Stretcher -> Rear Right Leg** | Joint: interlocking | Note: Right tenon inserted into rear-right leg's inner mortise. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-chair_ladder/harness.ts b/eval/tasks/muse-chair_ladder/harness.ts new file mode 100644 index 000000000..0cd243789 --- /dev/null +++ b/eval/tasks/muse-chair_ladder/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_ladder/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_ladder' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_ladder'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_ladder/prompt.md b/eval/tasks/muse-chair_ladder/prompt.md new file mode 100644 index 000000000..3aecb6ac4 --- /dev/null +++ b/eval/tasks/muse-chair_ladder/prompt.md @@ -0,0 +1,111 @@ +# chair_ladder (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a ladder-back style wooden chair designed for single-person seating, featuring a robust mortise-and-tenon assembly structure. + +## Geometry and Dimensions +Approx. 420.0 mm × 400.0 mm × 900.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating, load-bearing support, and backrest leaning torque resistance. + +## Structural Features +Seat panel; two front legs; two rear posts (extending to form the backrest frame); three horizontal back slats. + +## Special Requirements +Keep assembly split unchanged. Ensure exported STEP remains a closed solid. + +## Planned Component Quantity +8 + +## Component Names +- seat_panel +- front_left_leg +- front_right_leg +- rear_left_post +- rear_right_post +- back_slat_01 +- back_slat_02 +- back_slat_03 + +## Adjustable Parameters +- **width**: 420 (300.0 ~ 650.0 mm). Determines the seating area width; constrains extreme values to prevent tipping caused by unbalanced length-to-width ratios. +- **depth**: 400 (300.0 ~ 600.0 mm). Determines the seating area depth. +- **seat_height**: 450 (350.0 ~ 520.0 mm). Strictly follows ergonomic standards for single-person seating posture. +- **post_height**: 900 (750.0 ~ 1150.0 mm). Determines the overall height of the backrest to provide adequate lumbar and shoulder support. +- **leg_thickness**: 40 (20.0 ~ 70.0 mm). Lower limit ensures load-bearing stiffness; upper limit prevents interference and material waste. +- **seat_thickness**: 30 (15.0 ~ 50.0 mm). Must be thick enough to accommodate the insertion depth of the front leg tenons and support the user's weight. +- **slat_height**: 45 (20.0 ~ 80.0 mm). Defines the vertical contact area of the backrest slats for ergonomic comfort. +- **slat_thickness**: 15 (8.0 ~ 30.0 mm). Ensures structural strength of the slats against leaning forces. +- **slat_spacing**: 30 (15.0 ~ 60.0 mm). Controls the vertical gap between back slats for aesthetic proportion and weight reduction. +- **tenon_length**: 20 (8.0 ~ 40.0 mm). Determines the bite depth of the physical connections for legs and slats. +- **tenon_offset**: 5 (2.0 ~ 20.0 mm). Controls the setback distance of the tenon relative to the part edge to prevent wood splitting during assembly. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub of the chair. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets and through-holes) for the front legs and rear posts. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = seat\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features two rectangular sockets for front legs; rear features two oversized through-holes for the rear posts. + +### 2~3. front_left_leg & front_right_leg +The front supporting entities of the chair. +* **Component Purpose**: Vertical support. Transfers the front seat load to the ground, ensuring anti-overturning stability. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of length `tenon_length` that interference-fits into the bottom sockets of the seat panel. + +### 4~5. rear_left_post & rear_right_post +The rear supporting and backrest framing entities. +* **Component Purpose**: Vertical support and backrest frame. Transfers the rear seat load to the ground and provides mortise interfaces for the horizontal back slats. +* **Assembly Direction**: Passes vertically through the seat panel along the Z axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Passes through the seat panel's rear holes; inner faces feature three mortise sockets to receive the back slats. + +### 6~8. back_slat_01, back_slat_02, back_slat_03 +The horizontal functional support entities of the backrest. +* **Component Purpose**: Provides horizontal back support for human-computer interaction, connecting the two rear posts to form a rigid ladder-back structure. +* **Assembly Direction**: Inserted horizontally along the X axis into the rear posts. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both left and right ends feature tenons that insert into the corresponding mortises on the inner faces of the rear posts. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 8-component model: + +* **front_left_leg -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-left socket. +* **front_right_leg -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-right socket. +* **rear_left_post -> seat_panel** | Joint: interlocking | Note: Post passes through seat's rear-left through-hole. +* **rear_right_post -> seat_panel** | Joint: interlocking | Note: Post passes through seat's rear-right through-hole. +* **back_slat_01 -> rear_left_post & rear_right_post** | Joint: interlocking | Note: Top slat's left/right tenons inserted into top mortises of rear posts. +* **back_slat_02 -> rear_left_post & rear_right_post** | Joint: interlocking | Note: Middle slat's left/right tenons inserted into middle mortises of rear posts. +* **back_slat_03 -> rear_left_post & rear_right_post** | Joint: interlocking | Note: Bottom slat's left/right tenons inserted into bottom mortises of rear posts. diff --git a/eval/tasks/muse-chair_split_back/harness.ts b/eval/tasks/muse-chair_split_back/harness.ts new file mode 100644 index 000000000..5e3d63338 --- /dev/null +++ b/eval/tasks/muse-chair_split_back/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_split_back/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_split_back' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_split_back'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_split_back/prompt.md b/eval/tasks/muse-chair_split_back/prompt.md new file mode 100644 index 000000000..646e74644 --- /dev/null +++ b/eval/tasks/muse-chair_split_back/prompt.md @@ -0,0 +1,109 @@ +# chair_split_back (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged dining chair with a split-slat backrest designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 420.0 mm × 400.0 mm × 850.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; two front legs; two rear legs (posts); upper back slat; lower back slat. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- Seat panel +- Front left leg +- Front right leg +- Rear left leg +- Rear right leg +- Upper back slat +- Lower back slat + +## Adjustable Parameters +- **width**: 420 (300.0 ~ 650.0 mm). Controls the overall width of the chair seat and backrest span. +- **depth**: 400 (300.0 ~ 600.0 mm). Controls the seating depth. +- **seat_height**: 450 (350.0 ~ 520.0 mm). Determines the ergonomic seating height from the floor. +- **post_height**: 850 (700.0 ~ 1100.0 mm). Determines the total height of the rear legs/backrest posts. +- **leg_thickness**: 40 (20.0 ~ 70.0 mm). Defines the structural thickness of the legs and posts. +- **seat_thickness**: 30 (15.0 ~ 50.0 mm). Defines the thickness of the main load-bearing seat panel. +- **slat_height**: 60 (30.0 ~ 120.0 mm). Determines the vertical size of the backrest support slats. +- **slat_gap**: 40 (15.0 ~ 80.0 mm). Controls the vertical spacing between the upper and lower back slats. +- **tenon_length**: 20 (8.0 ~ 40.0 mm). Determines the insertion depth for the mortise and tenon joints. +- **tenon_offset**: 5 (2.0 ~ 20.0 mm). Controls the setback distance of the tenons relative to the part edge to prevent wood splitting. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the chair. +* **Component Purpose**: Acts as the main load-bearing base, providing blind sockets for the front legs and through-holes for the rear legs. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = `seat_height`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features two blind rectangular sockets at the front; rear features two through-holes for the back posts. + +### 2~3. Front Legs (Front Left, Front Right) +The front supporting entities of the chair. +* **Component Purpose**: Vertical support. Transfers the front seat load to the ground. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of length `tenon_length` that interference-fits into the bottom blind sockets of the seat panel. + +### 4~5. Rear Legs (Rear Left, Rear Right) +The rear supporting entities and backrest posts. +* **Component Purpose**: Vertical support and backrest frame. Transfers the rear seat load to the ground and provides mounting points for the back slats. +* **Assembly Direction**: Inserted upwards along the +Z axis through the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Passes entirely through the seat panel's rear holes. The inner X-faces feature mortise sockets to receive the back slats. + +### 6~7. Back Slats (Upper, Lower) +The functional support entities of the backrest. +* **Component Purpose**: Horizontal guide and lumbar/back support for human-computer interaction. +* **Assembly Direction**: Inserted horizontally along the X axis between the rear legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both left and right ends feature tenons that insert into the corresponding mortise sockets on the inner faces of the rear legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **Front Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-left blind socket. +* **Front Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-right blind socket. +* **Rear Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg passes through the seat's rear-left through-hole. +* **Rear Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg passes through the seat's rear-right through-hole. +* **Upper Back Slat -> Rear Left Leg & Rear Right Leg** | Joint: interlocking | Note: Slat tenons inserted into the upper inner mortises of the rear legs. +* **Lower Back Slat -> Rear Left Leg & Rear Right Leg** | Joint: interlocking | Note: Slat tenons inserted into the lower inner mortises of the rear legs. +* **Seat Panel -> All Leg Components** | Joint: Support Base | Note: Acts as the core hub; connection sockets and through-holes generated via boolean cut. diff --git a/eval/tasks/muse-chair_stretcher/harness.ts b/eval/tasks/muse-chair_stretcher/harness.ts new file mode 100644 index 000000000..0811e67b1 --- /dev/null +++ b/eval/tasks/muse-chair_stretcher/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-chair_stretcher/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'chair_stretcher' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-chair_stretcher'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-chair_stretcher/prompt.md b/eval/tasks/muse-chair_stretcher/prompt.md new file mode 100644 index 000000000..9e2d47d80 --- /dev/null +++ b/eval/tasks/muse-chair_stretcher/prompt.md @@ -0,0 +1,115 @@ +# chair_stretcher (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged dining chair with a backrest and reinforcing stretchers designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 420.0 mm × 400.0 mm × 860.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating with enhanced structural stability provided by lower stretchers. + +## Structural Features +Seat panel; four legs; backrest panel; four stretchers (front, rear, left, right). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +10 + +## Component Names +- Seat panel +- Front left leg +- Front right leg +- Rear left leg +- Rear right leg +- Backrest panel +- Front stretcher +- Rear stretcher +- Left stretcher +- Right stretcher + +## Adjustable Parameters +- **width**: 420 (300.0 ~ 700.0 mm). Determines the overall width of the chair seat and backrest. +- **depth**: 400 (300.0 ~ 700.0 mm). Determines the seating depth. +- **seat_height**: 450 (350.0 ~ 520.0 mm). Strictly follows ergonomic standards for single-person seating posture. +- **backrest_height**: 380 (250.0 ~ 600.0 mm). Provides adequate lumbar and back support without raising the center of gravity too high. +- **leg_thickness**: 40 (20.0 ~ 80.0 mm). Lower limit ensures load-bearing stiffness; upper limit prevents interference and material waste. +- **seat_thickness**: 30 (15.0 ~ 60.0 mm). Must be thick enough to accommodate the insertion depth of the leg and backrest tenons. +- **tenon_length**: 20 (8.0 ~ 15.0 mm). Determines the bite depth of the physical connections. +- **tenon_offset**: 5 (2.0 ~ 20.0 mm). Controls the setback distance of the tenon relative to the part edge to prevent wood splitting. +- **stretcher_height**: 150 (80.0 ~ 300.0 mm). Sets the vertical position of the stretchers from the ground to prevent leg splay. +- **stretcher_thickness**: 20 (10.0 ~ 40.0 mm). Determines the robustness of the horizontal bracing. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the chair. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the legs and backrest. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = seat\_height + seat\_thickness / 2.0$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four rectangular sockets for the legs; rear features a long slot for the backrest. + +### 2~5. Four Legs (Front Left, Front Right, Rear Left, Rear Right) +The supporting entities of the chair. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Contains mortises to receive the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. Inner faces feature mortises for the stretchers. + +### 6. Backrest Panel +The functional support entity of the chair. +* **Component Purpose**: Vertical guide. Provides back support for human-computer interaction, ensuring structural strength under large torque via a long mortise-and-tenon joint. +* **Assembly Direction**: Pressed downwards along the -Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features a full-width strip tenon inserted into the dedicated long socket at the rear of the seat panel. + +### 7~10. Four Stretchers (Front, Rear, Left, Right) +The horizontal bracing entities of the chair. +* **Component Purpose**: Connects the legs horizontally to prevent splaying, significantly increasing the overall structural rigidity and shear resistance of the base. +* **Assembly Direction**: Inserted horizontally along the X axis (front/rear) or Y axis (left/right) into the legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the inner faces of the legs. Left/right stretchers are vertically offset from front/rear stretchers to prevent internal tenon collision. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 10-component model: + +* **Front Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-left socket. +* **Front Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's front-right socket. +* **Rear Left Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-left socket. +* **Rear Right Leg -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's rear-right socket. +* **Backrest Panel -> Seat Panel** | Joint: interlocking | Note: Backrest bottom tenon inserted into seat's rear socket. +* **Front Stretcher -> Front Left & Right Legs** | Joint: interlocking | Note: Stretcher tenons inserted into inner X-faces of front legs. +* **Rear Stretcher -> Rear Left & Right Legs** | Joint: interlocking | Note: Stretcher tenons inserted into inner X-faces of rear legs. +* **Left Stretcher -> Front & Rear Left Legs** | Joint: interlocking | Note: Stretcher tenons inserted into inner Y-faces of left legs. +* **Right Stretcher -> Front & Rear Right Legs** | Joint: interlocking | Note: Stretcher tenons inserted into inner Y-faces of right legs. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all primary vertical connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/harness.ts b/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/harness.ts new file mode 100644 index 000000000..a4debbafa --- /dev/null +++ b/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_shoe_rack_narrow_three_tier' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_shoe_rack_narrow_three_tier'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/prompt.md b/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/prompt.md new file mode 100644 index 000000000..fdbaf802f --- /dev/null +++ b/eval/tasks/muse-cnc_shoe_rack_narrow_three_tier/prompt.md @@ -0,0 +1,97 @@ +# cnc_shoe_rack_narrow_three_tier (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a narrow three-tier shoe rack designed for small apartments, featuring closely spaced solid shelves for efficient storage. + +## Geometry and Dimensions +Approx. 640.0 mm × 290.0 mm × 720.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage for footwear. + +## Structural Features +Left side panel; right side panel; three solid shelf panels. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +5 + +## Component Names +- left_side_panel +- right_side_panel +- shelf_panel_01 +- shelf_panel_02 +- shelf_panel_03 + +## Adjustable Parameters +- **width**: 640.0 (540.0 ~ 780.0 mm). Controls the overall span of the rack and determines the horizontal storage capacity. +- **depth**: 290.0 (190.0 ~ 430.0 mm). Determines the footprint of the rack and the maximum shoe size it can accommodate. +- **height**: 720.0 (660.0 ~ 800.0 mm). Defines the total vertical space occupied by the rack. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the material thickness of the vertical side panels, ensuring structural stability. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Determines the width of the tenon joints connecting the shelves to the side panels, affecting joint strength. +- **shelf_depth**: 250.0 (150.0 ~ 390.0 mm). Defines the depth of the individual horizontal shelves. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the material thickness of the shelves to prevent sagging under the weight of footwear. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. left_side_panel +The vertical support structure on the left side. +* **Component Purpose**: Acts as the main load-bearing vertical support and provides mortise slots for the insertion of the shelves. +* **Assembly Direction**: Vertical placement along the Z-axis, positioned at the left extreme of the X-axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features three horizontal slots (mortises) cut into the inner face to receive the shelf tabs. + +### 2. right_side_panel +The vertical support structure on the right side. +* **Component Purpose**: Acts as the main load-bearing vertical support and provides mortise slots for the insertion of the shelves. +* **Assembly Direction**: Vertical placement along the Z-axis, positioned at the right extreme of the X-axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features three horizontal slots (mortises) cut into the inner face to receive the shelf tabs. + +### 3~5. shelf_panel_01, shelf_panel_02, shelf_panel_03 +The horizontal storage platforms. +* **Component Purpose**: Provides the flat surfaces for storing shoes and structurally ties the two side panels together to prevent lateral sway. +* **Assembly Direction**: Inserted horizontally along the X-axis between the left and right side panels at designated Z-heights. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Both the left and right ends feature protruding tabs (tenons) that fit into the corresponding slots of the side panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **shelf_panel_01 -> left_side_panel** | Joint: interlocking | Note: Left tab of the bottom shelf inserted into the lower slot of the left side panel. +* **shelf_panel_01 -> right_side_panel** | Joint: interlocking | Note: Right tab of the bottom shelf inserted into the lower slot of the right side panel. +* **shelf_panel_02 -> left_side_panel** | Joint: interlocking | Note: Left tab of the middle shelf inserted into the middle slot of the left side panel. +* **shelf_panel_02 -> right_side_panel** | Joint: interlocking | Note: Right tab of the middle shelf inserted into the middle slot of the right side panel. +* **shelf_panel_03 -> left_side_panel** | Joint: interlocking | Note: Left tab of the top shelf inserted into the upper slot of the left side panel. +* **shelf_panel_03 -> right_side_panel** | Joint: interlocking | Note: Right tab of the top shelf inserted into the upper slot of the right side panel. diff --git a/eval/tasks/muse-cnc_table_dining_trestle/harness.ts b/eval/tasks/muse-cnc_table_dining_trestle/harness.ts new file mode 100644 index 000000000..30beac183 --- /dev/null +++ b/eval/tasks/muse-cnc_table_dining_trestle/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_table_dining_trestle/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_table_dining_trestle' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_table_dining_trestle'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_table_dining_trestle/prompt.md b/eval/tasks/muse-cnc_table_dining_trestle/prompt.md new file mode 100644 index 000000000..3353f84c6 --- /dev/null +++ b/eval/tasks/muse-cnc_table_dining_trestle/prompt.md @@ -0,0 +1,105 @@ +# cnc_table_dining_trestle (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a dining trestle table with two panel legs and a low center stretcher designed for wood-based CNC assembly. + +## Geometry and Dimensions +Approx. 1600.0 mm × 820.0 mm × 750.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Multi-person dining and load-bearing surface. + +## Structural Features +Top panel; left leg panel; right leg panel; center stretcher. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +4 + +## Component Names +- Top panel +- Left leg panel +- Right leg panel +- Center stretcher + +## Adjustable Parameters +- **width**: 1600.0 (1500.0 ~ 1740.0 mm). Determines the overall length of the table surface. +- **depth**: 820.0 (720.0 ~ 960.0 mm). Determines the front-to-back depth of the table top. +- **height**: 750.0 (690.0 ~ 830.0 mm). Standard dining table height for ergonomic seating. +- **top_thickness**: 24.0 (18.0 ~ 32.0 mm). Ensures load-bearing stiffness for the main dining surface. +- **support_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the leg panels and stretcher, ensuring structural stability. +- **support_depth**: 620.0 (520.0 ~ 760.0 mm). Depth of the leg panels at the base to prevent tipping in the Y-axis. +- **support_span**: 1040.0 (940.0 ~ 1180.0 mm). Distance between the two leg panels, defining the seating clearance underneath. +- **corner_radius**: 18.0 (6.0 ~ 38.0 mm). Rounds the corners of the top panel for safety and aesthetics. +- **tab_width**: 20.0 (100.0 ~ 160.0 mm). Width of the tenons for assembly joints, determining the bite area of the physical connections. +- **stretcher_z**: 112.0 (52.0 ~ 192.0 mm). Vertical position of the center stretcher from the ground. +- **stretcher_depth**: 72.0 (100.0 ~ 212.0 mm). Width/depth of the stretcher beam. +- **stretcher_height**: 56.0 (40.0 ~ 136.0 mm). Height of the stretcher beam for longitudinal rigidity. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Top Panel +The central functional surface of the table. +* **Component Purpose**: Acts as the main load-bearing dining surface and provides localization references and mechanical interfaces (mortises/slots) for the leg panels. +* **Assembly Direction**: Fixed top component, positioned horizontally at absolute $Z = height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features slots to receive the top tabs of the leg panels. + +### 2. Left Leg Panel +The left supporting entity of the table. +* **Component Purpose**: Vertical support. Transfers the table load to the ground, ensuring anti-overturning stability in the Y-Z plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features tabs (tenons) that interference-fit into the bottom slots of the top panel. The inner face features a slot to receive the center stretcher. + +### 3. Right Leg Panel +The right supporting entity of the table. +* **Component Purpose**: Vertical support. Transfers the table load to the ground, ensuring anti-overturning stability in the Y-Z plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features tabs (tenons) that interference-fit into the bottom slots of the top panel. The inner face features a slot to receive the center stretcher. + +### 4. Center Stretcher +The horizontal tie beam connecting the two legs. +* **Component Purpose**: Prevents the leg panels from splaying and adds longitudinal rigidity to the table structure. +* **Assembly Direction**: Inserted horizontally along the X axis between the left and right leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tabs (tenons) that insert into the corresponding slots on the inner faces of the leg panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 4-component model: + +* **Left Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tabs inserted into top panel's left slots. +* **Right Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tabs inserted into top panel's right slots. +* **Center Stretcher -> Left Leg Panel** | Joint: interlocking | Note: Stretcher left tab inserted into left leg panel's inner slot. +* **Center Stretcher -> Right Leg Panel** | Joint: interlocking | Note: Stretcher right tab inserted into right leg panel's inner slot. diff --git a/eval/tasks/muse-cnc_table_square_side/harness.ts b/eval/tasks/muse-cnc_table_square_side/harness.ts new file mode 100644 index 000000000..53a5b5358 --- /dev/null +++ b/eval/tasks/muse-cnc_table_square_side/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_table_square_side/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_table_square_side' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_table_square_side'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_table_square_side/prompt.md b/eval/tasks/muse-cnc_table_square_side/prompt.md new file mode 100644 index 000000000..522cbcfaa --- /dev/null +++ b/eval/tasks/muse-cnc_table_square_side/prompt.md @@ -0,0 +1,117 @@ +# cnc_table_square_side (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a square side table with a centered lower shelf and compact panel legs, designed for CNC-machined wood assembly. + +## Geometry and Dimensions +Approx. 620.0 mm × 620.0 mm × 560.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage and display (side table usage). + +## Structural Features +Top panel; left leg panel; right leg panel; center stretcher; lower shelf. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +5 + +## Component Names +- Top panel +- Left leg panel +- Right leg panel +- Center stretcher +- Lower shelf + +## Adjustable Parameters +- **width**: 620.0 (520.0 ~ 760.0 mm). Determines the overall width of the table top. +- **depth**: 620.0 (520.0 ~ 760.0 mm). Determines the overall depth of the table top. +- **height**: 560.0 (500.0 ~ 640.0 mm). Determines the overall height of the table. +- **top_thickness**: 22.0 (16.0 ~ 30.0 mm). Thickness of the top panel, ensuring adequate load-bearing capacity. +- **support_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the leg panels and structural supports. +- **support_depth**: 420.0 (320.0 ~ 560.0 mm). Depth of the vertical leg panels. +- **support_span**: 360.0 (260.0 ~ 500.0 mm). Distance between the left and right leg panels. +- **corner_radius**: 14.0 (2.0 ~ 34.0 mm). Radius of the rounded corners on the top panel for safety and aesthetics. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Width of the connection tabs (tenons) for assembly. +- **stretcher_z**: 86.0 (40.0 ~ 166.0 mm). Vertical position of the center stretcher from the ground. +- **stretcher_depth**: 60.0 (100.0 ~ 200.0 mm). Depth of the center stretcher. +- **stretcher_height**: 48.0 (40.0 ~ 128.0 mm). Height of the center stretcher. +- **shelf_z**: 146.0 (86.0 ~ 226.0 mm). Vertical position of the lower shelf from the ground. +- **shelf_depth**: 280.0 (180.0 ~ 420.0 mm). Depth of the lower shelf. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the lower shelf. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Top Panel +The main horizontal surface of the table. +* **Component Purpose**: Acts as the primary load-bearing surface for placing items and serves as the top structural hub connecting the leg panels. +* **Assembly Direction**: Fixed top component, positioned at absolute $Z = height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features through-slots (mortises) to receive the top tabs of the leg panels. + +### 2. Left Leg Panel +The left vertical support entity. +* **Component Purpose**: Vertical support. Transfers the table load to the ground and provides structural slots for the stretcher and lower shelf. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tenons for the top panel, and mortises for the stretcher and shelf. + +### 3. Right Leg Panel +The right vertical support entity. +* **Component Purpose**: Vertical support. Transfers the table load to the ground and provides structural slots for the stretcher and lower shelf. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tenons for the top panel, and mortises for the stretcher and shelf. + +### 4. Center Stretcher +The lower structural tie. +* **Component Purpose**: Horizontal structural beam connecting the lower parts of the legs to prevent lateral sway and increase overall rigidity. +* **Assembly Direction**: Horizontal insertion along the X axis between the leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features end tenons that fit into the corresponding lower slots of the leg panels. + +### 5. Lower Shelf +The secondary horizontal surface. +* **Component Purpose**: Provides additional storage space and acts as a secondary structural tie between the legs. +* **Assembly Direction**: Horizontal insertion along the X axis between the leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features end tenons that fit into the corresponding middle slots of the leg panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **Left Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tenons inserted into top panel's left slots. +* **Right Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tenons inserted into top panel's right slots. +* **Center Stretcher -> Left Leg Panel** | Joint: interlocking | Note: Stretcher left tenon inserted into left leg's lower slot. +* **Center Stretcher -> Right Leg Panel** | Joint: interlocking | Note: Stretcher right tenon inserted into right leg's lower slot. +* **Lower Shelf -> Left Leg Panel** | Joint: interlocking | Note: Shelf left tenon inserted into left leg's middle slot. +* **Lower Shelf -> Right Leg Panel** | Joint: interlocking | Note: Shelf right tenon inserted into right leg's middle slot. diff --git a/eval/tasks/muse-cnc_table_study_shelf/harness.ts b/eval/tasks/muse-cnc_table_study_shelf/harness.ts new file mode 100644 index 000000000..265dfd6ae --- /dev/null +++ b/eval/tasks/muse-cnc_table_study_shelf/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_table_study_shelf/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_table_study_shelf' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_table_study_shelf'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_table_study_shelf/prompt.md b/eval/tasks/muse-cnc_table_study_shelf/prompt.md new file mode 100644 index 000000000..e5f3c5c7e --- /dev/null +++ b/eval/tasks/muse-cnc_table_study_shelf/prompt.md @@ -0,0 +1,117 @@ +# cnc_table_study_shelf (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a study table with panel legs, a lower shelf, and a simple center stretcher designed for flat-pack CNC wood manufacturing. + +## Geometry and Dimensions +Approx. 1360.0 mm × 660.0 mm × 742.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing work surface for studying, writing, and supporting computer equipment. + +## Structural Features +Top panel; left leg panel; right leg panel; center stretcher; lower shelf. + +## Special Requirements +Keep assembly split unchanged. Ensure all internal corners for mortises account for CNC tool radius offsets (dog-bones or overcuts) if machined physically. + +## Planned Component Quantity +5 + +## Component Names +- Top panel +- Left leg panel +- Right leg panel +- Center stretcher +- Lower shelf + +## Adjustable Parameters +- **width**: 1360.0 (1260.0 ~ 1500.0 mm). Overall width of the table top. +- **depth**: 660.0 (560.0 ~ 800.0 mm). Overall depth of the table top. +- **height**: 742.0 (682.0 ~ 822.0 mm). Total height of the table from the ground to the top surface. +- **top_thickness**: 22.0 (16.0 ~ 30.0 mm). Thickness of the main work surface, ensuring adequate load-bearing stiffness. +- **support_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the leg panels and structural supports. +- **support_depth**: 520.0 (420.0 ~ 660.0 mm). Depth of the leg panels, providing anti-overturning stability in the Y-Z plane. +- **support_span**: 960.0 (860.0 ~ 1100.0 mm). Distance between the left and right leg panels. +- **corner_radius**: 12.0 (0.0 ~ 32.0 mm). Radius for the rounded corners of the top panel to prevent sharp edge injuries. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Width of the tenon tabs for assembly joints. +- **stretcher_z**: 108.0 (48.0 ~ 188.0 mm). Vertical position of the center stretcher from the ground. +- **stretcher_depth**: 68.0 (100.0 ~ 208.0 mm). Depth (width) of the center stretcher. +- **stretcher_height**: 54.0 (40.0 ~ 134.0 mm). Height of the center stretcher. +- **shelf_z**: 178.0 (118.0 ~ 258.0 mm). Vertical position of the lower shelf from the ground. +- **shelf_depth**: 360.0 (260.0 ~ 500.0 mm). Depth of the lower shelf. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the lower shelf panel. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Top Panel +The primary work surface of the table. +* **Component Purpose**: Acts as the main horizontal load-bearing surface and provides localization references (mortise slots) for the leg panels. +* **Assembly Direction**: Placed downwards along the -Z axis onto the leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom face features mortise slots to receive the top tabs of the leg panels. + +### 2. Left Leg Panel +The left vertical support structure. +* **Component Purpose**: Transfers the table load to the ground, ensuring stability. Provides mortise slots for the stretcher and lower shelf. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tabs (tenons) that fit into the top panel, and mortise slots on its inner face for horizontal components. + +### 3. Right Leg Panel +The right vertical support structure. +* **Component Purpose**: Transfers the table load to the ground, ensuring stability. Provides mortise slots for the stretcher and lower shelf. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tabs (tenons) that fit into the top panel, and mortise slots on its inner face for horizontal components. + +### 4. Center Stretcher +The primary horizontal structural tie. +* **Component Purpose**: Connects the left and right leg panels near the base to prevent racking and ensure lateral stability in the X-Z plane. +* **Assembly Direction**: Horizontal insertion along the X axis between the leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenon tabs that insert into the corresponding slots on the leg panels. + +### 5. Lower Shelf +The secondary horizontal surface. +* **Component Purpose**: Provides additional storage space below the main table top and acts as a secondary structural tie to reinforce the leg panels. +* **Assembly Direction**: Horizontal insertion along the X axis between the leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenon tabs that insert into the corresponding slots on the leg panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **Left Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tabs inserted into top panel's left slots. +* **Right Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tabs inserted into top panel's right slots. +* **Center Stretcher -> Left Leg Panel** | Joint: interlocking | Note: Stretcher left tab inserted into left leg's lower slot. +* **Center Stretcher -> Right Leg Panel** | Joint: interlocking | Note: Stretcher right tab inserted into right leg's lower slot. +* **Lower Shelf -> Left Leg Panel** | Joint: interlocking | Note: Shelf left tabs inserted into left leg's middle slots. +* **Lower Shelf -> Right Leg Panel** | Joint: interlocking | Note: Shelf right tabs inserted into right leg's middle slots. diff --git a/eval/tasks/muse-cnc_table_workbench_wide/harness.ts b/eval/tasks/muse-cnc_table_workbench_wide/harness.ts new file mode 100644 index 000000000..2a5812313 --- /dev/null +++ b/eval/tasks/muse-cnc_table_workbench_wide/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_table_workbench_wide/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_table_workbench_wide' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_table_workbench_wide'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_table_workbench_wide/prompt.md b/eval/tasks/muse-cnc_table_workbench_wide/prompt.md new file mode 100644 index 000000000..98d9b059d --- /dev/null +++ b/eval/tasks/muse-cnc_table_workbench_wide/prompt.md @@ -0,0 +1,127 @@ +# cnc_table_workbench_wide (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a wide, heavy-duty wooden workbench featuring panel legs, a center stretcher, and a lower utility shelf, designed for CNC-machined flat-pack assembly. + +## Geometry and Dimensions +Approx. 1800.0 mm × 760.0 mm × 760.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing work surface and utility storage; suitable for workshop or heavy-duty multi-purpose tasks. + +## Structural Features +Top panel; two leg panels (left and right); center stretcher; lower shelf; front apron; rear apron. + +## Special Requirements +Keep assembly split unchanged. All tab-and-slot (mortise and tenon) features must maintain appropriate tolerances for interference fitting without requiring external hardware. + +## Planned Component Quantity +7 + +## Component Names +- top_panel +- left_leg_panel +- right_leg_panel +- center_stretcher +- lower_shelf +- front_apron +- rear_apron + +## Adjustable Parameters +- **width**: 1800.0 (1700.0 ~ 1940.0 mm). Determines the overall span of the workbench. +- **depth**: 760.0 (660.0 ~ 900.0 mm). Determines the working surface depth. +- **height**: 760.0 (700.0 ~ 840.0 mm). Ergonomic height for a standing or seated work table. +- **top_thickness**: 24.0 (18.0 ~ 32.0 mm). Ensures sufficient load-bearing capacity and stiffness for the main work surface. +- **support_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the vertical legs and structural panels. +- **support_depth**: 600.0 (500.0 ~ 740.0 mm). Depth of the leg panels, providing front-to-back stability. +- **support_span**: 1220.0 (1120.0 ~ 1360.0 mm). Distance between the two leg panels, defining the unsupported span of the top panel. +- **corner_radius**: 0.0 (0.0 ~ 20.0 mm). Controls the edge rounding of the top panel for safety. +- **tab_width**: 20.0 (100.0 ~ 160.0 mm). Width of the mortise and tenon joints connecting the panels. +- **stretcher_z**: 108.0 (48.0 ~ 188.0 mm). Vertical position of the center stretcher from the ground. +- **shelf_z**: 180.0 (120.0 ~ 260.0 mm). Vertical position of the lower utility shelf. +- **apron_z**: 620.0 (560.0 ~ 700.0 mm). Vertical position of the aprons supporting the top panel. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The main horizontal work surface of the workbench. +* **Component Purpose**: Provides the primary load-bearing area for work tasks and acts as the upper locking hub for the leg panels and aprons. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the leg panels and aprons. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features rectangular mortises (slots) on its underside to receive the tenons from the legs and aprons. + +### 2. left_leg_panel +The left vertical supporting entity. +* **Component Purpose**: Transfers the load from the top panel to the ground. Features cutouts (windows) to reduce weight and slots to receive the horizontal stretchers, shelves, and aprons. +* **Assembly Direction**: Vertical support, positioned on the left side of the assembly. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top edge features tenons that insert into the top panel; face features slots for horizontal components. + +### 3. right_leg_panel +The right vertical supporting entity. +* **Component Purpose**: Transfers the load from the top panel to the ground. Mirrors the left leg panel in function and connectivity. +* **Assembly Direction**: Vertical support, positioned on the right side of the assembly. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top edge features tenons that insert into the top panel; face features slots for horizontal components. + +### 4. center_stretcher +The lower horizontal stabilizing beam. +* **Component Purpose**: Connects the lower portion of the leg panels to prevent lateral racking and increase overall structural stability. +* **Assembly Direction**: Inserted horizontally along the X axis between the left and right leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Ends feature tenons that lock into the lower slots of the leg panels. + +### 5. lower_shelf +The horizontal utility panel. +* **Component Purpose**: Provides utility storage space beneath the main work surface and acts as an additional structural tie between the legs. +* **Assembly Direction**: Inserted horizontally along the X axis between the left and right leg panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Ends feature tenons that lock into the mid-lower slots of the leg panels. + +### 6. front_apron +The front vertical support beam under the top panel. +* **Component Purpose**: Prevents the top panel from sagging under heavy loads and adds lateral stability to the upper frame. +* **Assembly Direction**: Inserted horizontally along the X axis between the leg panels, interfacing with the underside of the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Ends insert into the leg panels; top edge features tenons that insert into the top panel. + +### 7. rear_apron +The rear vertical support beam under the top panel. +* **Component Purpose**: Mirrors the front apron, preventing top panel sag and adding lateral stability to the rear of the upper frame. +* **Assembly Direction**: Inserted horizontally along the X axis between the leg panels, interfacing with the underside of the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Ends insert into the leg panels; top edge features tenons that insert into the top panel. + +--- + +## Component Assembly Graph (Textual) +* **left_leg_panel -> top_panel** | Joint: interlocking | Note: Leg top tenons inserted into the top panel's left-side slots. +* **right_leg_panel -> top_panel** | Joint: interlocking | Note: Leg top tenons inserted into the top panel's right-side slots. +* **center_stretcher -> left_leg_panel & right_leg_panel** | Joint: interlocking | Note: Stretcher ends inserted into the lower slots of both leg panels. +* **lower_shelf -> left_leg_panel & right_leg_panel** | Joint: interlocking | Note: Shelf ends inserted into the mid-level slots of both leg panels. +* **front_apron -> left_leg_panel & right_leg_panel** | Joint: interlocking | Note: Apron ends inserted into the upper-front slots of both leg panels. +* **rear_apron -> left_leg_panel & right_leg_panel** | Joint: interlocking | Note: Apron ends inserted into the upper-rear slots of both leg panels. +* **front_apron & rear_apron -> top_panel** | Joint: interlocking | Note: Apron top tenons inserted into the top panel's underside slots. diff --git a/eval/tasks/muse-cnc_table_writing_desk/harness.ts b/eval/tasks/muse-cnc_table_writing_desk/harness.ts new file mode 100644 index 000000000..12b266329 --- /dev/null +++ b/eval/tasks/muse-cnc_table_writing_desk/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_table_writing_desk/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_table_writing_desk' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_table_writing_desk'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_table_writing_desk/prompt.md b/eval/tasks/muse-cnc_table_writing_desk/prompt.md new file mode 100644 index 000000000..644609c3e --- /dev/null +++ b/eval/tasks/muse-cnc_table_writing_desk/prompt.md @@ -0,0 +1,115 @@ +# cnc_table_writing_desk (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a writing desk with panel legs and front-back apron rails below the top, designed for flat-pack CNC-machined assembly. + +## Geometry and Dimensions +Approx. 1280.0 mm × 620.0 mm × 742.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person working and writing surface; load-bearing desk structure requiring lateral stability. + +## Structural Features +Top panel; left leg panel; right leg panel; front apron; rear apron. + +## Special Requirements +Keep assembly split unchanged. Ensure all inner corners of mortises account for CNC tool radius offsets (dog-bone fillets) if machined practically, though the base geometry assumes perfect boolean cuts. + +## Planned Component Quantity +5 + +## Component Names +- Top panel +- Left leg panel +- Right leg panel +- Front apron +- Rear apron + +## Adjustable Parameters +- **width**: 1280.0 (1180.0 ~ 1420.0 mm). Determines the overall span of the primary workspace. +- **depth**: 620.0 (520.0 ~ 760.0 mm). Determines the front-to-back working area. +- **height**: 742.0 (682.0 ~ 822.0 mm). Strictly follows ergonomic standards for a seated writing desk. +- **top_thickness**: 22.0 (16.0 ~ 30.0 mm). Ensures structural rigidity of the work surface and provides sufficient depth for blind or through mortises. +- **support_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the vertical leg panels and aprons, balancing load-bearing capacity and weight. +- **support_depth**: 520.0 (420.0 ~ 660.0 mm). Depth of the leg panels at the base, ensuring anti-overturning stability. +- **support_span**: 920.0 (820.0 ~ 1060.0 mm). The clear distance between the left and right leg panels, dictating legroom. +- **corner_radius**: 12.0 (0.0 ~ 32.0 mm). Rounds the corners of the top panel for user safety and aesthetics. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Width of the tenon tabs used for interlocking the panels. +- **apron_z**: 612.0 (552.0 ~ 692.0 mm). Vertical placement of the apron rails, ensuring adequate knee clearance while maintaining structural bracing. +- **apron_depth**: 40.0 (100.0 ~ 180.0 mm). The horizontal offset/positioning of the aprons relative to the desk edges. +- **apron_height**: 60.0 (40.0 ~ 140.0 mm). The vertical height of the apron panels to resist racking forces. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Top Panel +The central horizontal hub of the desk. +* **Component Purpose**: Acts as the main load-bearing work surface and provides localization references and mechanical interfaces (mortise slots) for the leg panels and aprons. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The underside features rectangular slots to receive the top tenons of the legs and aprons. + +### 2. Left Leg Panel +The primary vertical support on the left side. +* **Component Purpose**: Vertical support. Transfers the desk load to the ground and features cutouts (windows) for weight reduction and aesthetics. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tenons that interface with the top panel, and vertical slots to receive the side tenons of the aprons. + +### 3. Right Leg Panel +The primary vertical support on the right side. +* **Component Purpose**: Vertical support. Transfers the desk load to the ground, mirroring the left leg panel. +* **Assembly Direction**: Inserted upwards along the +Z axis into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top tenons that interface with the top panel, and vertical slots to receive the side tenons of the aprons. + +### 4. Front Apron +The structural cross-beam at the front of the desk. +* **Component Purpose**: Lateral bracing. Prevents side-to-side racking of the desk and stabilizes the two leg panels. +* **Assembly Direction**: Inserted horizontally along the Y axis between the leg panels, and upwards into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features tenons on the left/right ends to lock into the leg panels, and top tenons to lock into the top panel. + +### 5. Rear Apron +The structural cross-beam at the rear of the desk. +* **Component Purpose**: Lateral bracing. Works in tandem with the front apron to ensure complete rigidity of the desk frame. +* **Assembly Direction**: Inserted horizontally along the Y axis between the leg panels, and upwards into the top panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features tenons on the left/right ends to lock into the leg panels, and top tenons to lock into the top panel. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **Left Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tenons inserted into top panel's left-side slots. +* **Right Leg Panel -> Top Panel** | Joint: interlocking | Note: Leg top tenons inserted into top panel's right-side slots. +* **Front Apron -> Left Leg Panel** | Joint: interlocking | Note: Apron left tenon inserted into left leg's front slot. +* **Front Apron -> Right Leg Panel** | Joint: interlocking | Note: Apron right tenon inserted into right leg's front slot. +* **Rear Apron -> Left Leg Panel** | Joint: interlocking | Note: Apron left tenon inserted into left leg's rear slot. +* **Rear Apron -> Right Leg Panel** | Joint: interlocking | Note: Apron right tenon inserted into right leg's rear slot. +* **Front & Rear Aprons -> Top Panel** | Joint: interlocking | Note: Apron top tenons inserted into top panel's longitudinal slots. diff --git a/eval/tasks/muse-cnc_tv_stand_asym_media/harness.ts b/eval/tasks/muse-cnc_tv_stand_asym_media/harness.ts new file mode 100644 index 000000000..871200c93 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_asym_media/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_asym_media/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_asym_media' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_asym_media'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_asym_media/prompt.md b/eval/tasks/muse-cnc_tv_stand_asym_media/prompt.md new file mode 100644 index 000000000..7ef40bd3c --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_asym_media/prompt.md @@ -0,0 +1,142 @@ +# cnc_tv_stand_asym_media (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct an asymmetric media TV stand with two off-center dividers to create varied bay widths, designed for CNC-machined wood assembly. + +## Geometry and Dimensions +Approx. 1640.0 mm × 420.0 mm × 500.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage for media equipment and television displays. + +## Structural Features +Top panel; bottom panel; left side panel; right side panel; two center dividers; three shelf segments. + +## Special Requirements +Keep assembly split unchanged. Ensure all tab-and-slot (mortise and tenon) features account for tool radius offsets during CNC machining. + +## Planned Component Quantity +9 + +## Component Names +- top_panel +- bottom_panel +- left_side_panel +- right_side_panel +- center_divider_01 +- center_divider_02 +- shelf_l01_b01 +- shelf_l01_b02 +- shelf_l01_b03 + +## Adjustable Parameters +- **width**: 1640.0 (1540.0 ~ 1780.0 mm). Determines the overall horizontal span of the TV stand. +- **depth**: 420.0 (320.0 ~ 560.0 mm). Determines the footprint depth, ensuring stability and adequate space for media devices. +- **height**: 500.0 (440.0 ~ 580.0 mm). Sets the viewing height of the stand. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Standard sheet material thickness for vertical structural integrity. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the top load-bearing panel. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the bottom base panel. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Width of the mortise and tenon tabs used for interlocking the panels. +- **corner_radius**: 0.0 (0.0 ~ 20.0 mm). Softens the outer corners of the horizontal panels for safety and aesthetics. +- **shelf_depth**: 360.0 (260.0 ~ 500.0 mm). Depth of the internal storage shelves, slightly recessed from the main frame. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the internal shelf segments. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The main upper load-bearing surface. +* **Component Purpose**: Supports the television and ties the top of all vertical supports together to prevent lateral racking. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the vertical panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features slotted mortises on its underside to receive the top tabs of the side panels and dividers. + +### 2. bottom_panel +The main base surface. +* **Component Purpose**: Provides ground support, distributes the structural load, and ties the bottom of the vertical supports together. +* **Assembly Direction**: Fixed base component, positioned at the bottom of the assembly. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features slotted mortises to receive the bottom tabs of the side panels and dividers. + +### 3. left_side_panel +The left outer vertical support. +* **Component Purpose**: Bears the vertical load on the left extremity and encloses the stand. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs on the top and bottom edges, and mortise slots on the inner face for the shelf. + +### 4. right_side_panel +The right outer vertical support. +* **Component Purpose**: Bears the vertical load on the right extremity and encloses the stand. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs on the top and bottom edges, and mortise slots on the inner face for the shelf. + +### 5. center_divider_01 +The first internal vertical support. +* **Component Purpose**: Provides intermediate vertical support and divides the stand to create the first asymmetric bay. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top/bottom tabs and side slots for shelf insertion. + +### 6. center_divider_02 +The second internal vertical support. +* **Component Purpose**: Provides intermediate vertical support and divides the stand to create the second and third asymmetric bays. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top/bottom tabs and side slots for shelf insertion. + +### 7. shelf_l01_b01 +The horizontal storage surface for the first bay. +* **Component Purpose**: Provides internal storage space between the left side panel and the first center divider. +* **Assembly Direction**: Inserted horizontally along the X/Y axis into the vertical supports. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features side tabs that lock into the vertical panels. + +### 8. shelf_l01_b02 +The horizontal storage surface for the second bay. +* **Component Purpose**: Provides internal storage space between the two center dividers. +* **Assembly Direction**: Inserted horizontally along the X/Y axis into the vertical supports. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features side tabs that lock into the vertical dividers. + +### 9. shelf_l01_b03 +The horizontal storage surface for the third bay. +* **Component Purpose**: Provides internal storage space between the second center divider and the right side panel. +* **Assembly Direction**: Inserted horizontally along the X/Y axis into the vertical supports. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features side tabs that lock into the vertical panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **left_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's left-most slots. +* **right_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's right-most slots. +* **center_divider_01 -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's inner-left slots. +* **center_divider_02 -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's inner-right slots. +* **shelf_l01_b01 -> left_side_panel & center_divider_01** | Joint: interlocking | Note: Suspended horizontally between the left panel and first divider. +* **shelf_l01_b02 -> center_divider_01 & center_divider_02** | Joint: interlocking | Note: Suspended horizontally between the two center dividers. +* **shelf_l01_b03 -> center_divider_02 & right_side_panel** | Joint: interlocking | Note: Suspended horizontally between the second divider and right panel. +* **top_panel -> All Vertical Panels** | Joint: interlocking | Note: Top panel slots fit over the top tabs of the left side, right side, and both center dividers. diff --git a/eval/tasks/muse-cnc_tv_stand_compact_console/harness.ts b/eval/tasks/muse-cnc_tv_stand_compact_console/harness.ts new file mode 100644 index 000000000..9da4f79cb --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_compact_console/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_compact_console/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_compact_console' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_compact_console'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_compact_console/prompt.md b/eval/tasks/muse-cnc_tv_stand_compact_console/prompt.md new file mode 100644 index 000000000..07cdcfa4d --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_compact_console/prompt.md @@ -0,0 +1,130 @@ +# cnc_tv_stand_compact_console (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a compact TV console sized for smaller apartments, featuring one central divider and one shelf level, designed for flat-pack assembly. + +## Geometry and Dimensions +Approx. 1180.0 mm × 380.0 mm × 460.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage (supporting a television, media devices, and internal storage items). + +## Structural Features +Top panel; bottom panel; left side panel; right side panel; center divider; two shelf segments. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- top_panel +- bottom_panel +- left_side_panel +- right_side_panel +- center_divider_01 +- shelf_l01_b01 +- shelf_l01_b02 + +## Adjustable Parameters +- **width**: 1180.0 (1080.0 ~ 1320.0 mm). Controls the overall span of the TV stand to accommodate different screen sizes. +- **depth**: 380.0 (280.0 ~ 520.0 mm). Determines the footprint and internal storage capacity. +- **height**: 460.0 (400.0 ~ 540.0 mm). Sets the vertical elevation of the TV for optimal viewing angles. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). General panel thickness ensuring overall structural stiffness. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the top load-bearing panel to prevent sagging under the TV's weight. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the base panel for foundational stability. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Width of the interlocking tenons (tabs) for assembly connections. +- **corner_radius**: 10.0 (0.0 ~ 30.0 mm). Rounds the corners of the top panel for safety and aesthetics. +- **shelf_depth**: 330.0 (230.0 ~ 470.0 mm). Depth of the internal shelves, slightly recessed from the main frame. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the shelf panels to support media equipment without deformation. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The main upper surface of the console. +* **Component Purpose**: Acts as the primary load-bearing platform for the TV and ties the vertical supports together at the top. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the vertical panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features mortise slots on its underside to receive the top tenons of the side panels and divider. + +### 2. bottom_panel +The foundational base of the console. +* **Component Purpose**: Rests on the floor, providing a stable base and tying the vertical supports together at the bottom. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = 0$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features mortise slots on its top face to receive the bottom tenons of the side panels and divider. + +### 3. left_side_panel +The left vertical enclosure and support. +* **Component Purpose**: Encloses the left side of the console, transferring loads from the top panel to the bottom panel, and supporting the left shelf. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top and bottom tenons (tabs) and internal mortise slots for the shelf. + +### 4. right_side_panel +The right vertical enclosure and support. +* **Component Purpose**: Encloses the right side of the console, transferring loads from the top panel to the bottom panel, and supporting the right shelf. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top and bottom tenons (tabs) and internal mortise slots for the shelf. + +### 5. center_divider_01 +The central vertical support. +* **Component Purpose**: Divides the internal space into two bays and provides mid-span load-bearing support to prevent the top panel from sagging. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features top and bottom tenons, and mortise slots on both sides to support the inner edges of the shelves. + +### 6. shelf_l01_b01 +The horizontal storage platform in the left bay. +* **Component Purpose**: Provides a dedicated surface for media devices or storage within the left section. +* **Assembly Direction**: Inserted horizontally along the X/Y axis into the left panel and center divider. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features side tenons that lock into the corresponding slots of the vertical supports. + +### 7. shelf_l01_b02 +The horizontal storage platform in the right bay. +* **Component Purpose**: Provides a dedicated surface for media devices or storage within the right section. +* **Assembly Direction**: Inserted horizontally along the X/Y axis into the right panel and center divider. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features side tenons that lock into the corresponding slots of the vertical supports. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **left_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs of left panel inserted into bottom panel slots. +* **right_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs of right panel inserted into bottom panel slots. +* **center_divider_01 -> bottom_panel** | Joint: interlocking | Note: Bottom tabs of divider inserted into bottom panel slots. +* **shelf_l01_b01 -> left_side_panel** | Joint: interlocking | Note: Left tabs of shelf inserted into left panel slots. +* **shelf_l01_b01 -> center_divider_01** | Joint: interlocking | Note: Right tabs of shelf inserted into left side of the center divider. +* **shelf_l01_b02 -> center_divider_01** | Joint: interlocking | Note: Left tabs of shelf inserted into right side of the center divider. +* **shelf_l01_b02 -> right_side_panel** | Joint: interlocking | Note: Right tabs of shelf inserted into right panel slots. +* **top_panel -> left_side_panel** | Joint: interlocking | Note: Top tabs of left panel inserted into top panel slots. +* **top_panel -> right_side_panel** | Joint: interlocking | Note: Top tabs of right panel inserted into top panel slots. +* **top_panel -> center_divider_01** | Joint: interlocking | Note: Top tabs of divider inserted into top panel slots. diff --git a/eval/tasks/muse-cnc_tv_stand_low_two_bay/harness.ts b/eval/tasks/muse-cnc_tv_stand_low_two_bay/harness.ts new file mode 100644 index 000000000..b44a7e609 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_low_two_bay/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_low_two_bay/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_low_two_bay' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_low_two_bay'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_low_two_bay/prompt.md b/eval/tasks/muse-cnc_tv_stand_low_two_bay/prompt.md new file mode 100644 index 000000000..0029fda92 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_low_two_bay/prompt.md @@ -0,0 +1,130 @@ +# cnc_tv_stand_low_two_bay (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Low two-bay TV stand with one center divider and a single shelf level, designed for flat-pack assembly and media storage. + +## Geometry and Dimensions +Approx. 1380.0 mm × 400.0 mm × 460.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage (supporting a television on top and media devices on internal shelves). + +## Structural Features +Top panel; bottom panel; left side panel; right side panel; center divider; two shelf panels. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- Top panel +- Bottom panel +- Left side panel +- Right side panel +- Center divider 01 +- Shelf l01 b01 +- Shelf l01 b02 + +## Adjustable Parameters +- **width**: 1380.0 (1280.0 ~ 1520.0 mm). Determines the overall span of the TV stand. +- **depth**: 400.0 (300.0 ~ 540.0 mm). Determines the overall depth to accommodate various TV base sizes and media equipment. +- **height**: 460.0 (400.0 ~ 540.0 mm). Sets the viewing height of the television. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Controls the thickness of the vertical support panels (sides and divider) for structural stability. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Controls the thickness of the top load-bearing panel. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Controls the thickness of the bottom base panel. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Defines the width of the interlocking tenons for the assembly joints. +- **corner_radius**: 10.0 (0.0 ~ 30.0 mm). Defines the fillet radius on the corners of the top panel for aesthetics and safety. +- **shelf_depth**: 360.0 (260.0 ~ 500.0 mm). Determines the depth of the internal storage shelves. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Controls the load-bearing thickness of the internal shelves. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Top panel +The upper horizontal surface of the TV stand. +* **Component Purpose**: Acts as the primary load-bearing surface for the television and provides mortise slots to lock the vertical supports in place. +* **Assembly Direction**: Placed downwards along the -Z axis onto the vertical panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features slots on its underside to receive the top tabs of the side panels and center divider. + +### 2. Bottom panel +The lower horizontal base of the TV stand. +* **Component Purpose**: Acts as the foundational base, connecting the vertical panels at the bottom to ensure structural rigidity and ground contact. +* **Assembly Direction**: Positioned at the base, receiving vertical panels along the +Z axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features slots to receive the bottom tabs of the side panels and center divider. + +### 3. Left side panel +The left vertical support structure. +* **Component Purpose**: Transfers the load from the top panel to the bottom panel and encloses the left side of the stand. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features protruding tabs on the top and bottom edges, and mortise slots on the inner face for the shelf. + +### 4. Right side panel +The right vertical support structure. +* **Component Purpose**: Transfers the load from the top panel to the bottom panel and encloses the right side of the stand. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features protruding tabs on the top and bottom edges, and mortise slots on the inner face for the shelf. + +### 5. Center divider 01 +The central vertical support structure. +* **Component Purpose**: Divides the internal space into two bays and provides central vertical support to prevent the top panel from sagging under the TV's weight. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features tabs on the top and bottom, and mortise slots on both sides to support the inner edges of the shelves. + +### 6. Shelf l01 b01 +The horizontal storage surface in the left bay. +* **Component Purpose**: Provides a platform for media devices in the left compartment. +* **Assembly Direction**: Inserted horizontally between the left side panel and the center divider. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features tabs on its left and right edges that slot into the left side panel and center divider. + +### 7. Shelf l01 b02 +The horizontal storage surface in the right bay. +* **Component Purpose**: Provides a platform for media devices in the right compartment. +* **Assembly Direction**: Inserted horizontally between the center divider and the right side panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features tabs on its left and right edges that slot into the center divider and right side panel. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **Left side panel -> Bottom panel** | Joint: interlocking | Note: Bottom tabs of left panel inserted into bottom panel slots. +* **Right side panel -> Bottom panel** | Joint: interlocking | Note: Bottom tabs of right panel inserted into bottom panel slots. +* **Center divider 01 -> Bottom panel** | Joint: interlocking | Note: Bottom tabs of divider inserted into bottom panel slots. +* **Top panel -> Left side panel** | Joint: interlocking | Note: Top tabs of left panel inserted into top panel slots. +* **Top panel -> Right side panel** | Joint: interlocking | Note: Top tabs of right panel inserted into top panel slots. +* **Top panel -> Center divider 01** | Joint: interlocking | Note: Top tabs of divider inserted into top panel slots. +* **Shelf l01 b01 -> Left side panel** | Joint: interlocking | Note: Left tabs of shelf inserted into left side panel slots. +* **Shelf l01 b01 -> Center divider 01** | Joint: interlocking | Note: Right tabs of shelf inserted into center divider slots. +* **Shelf l01 b02 -> Center divider 01** | Joint: interlocking | Note: Left tabs of shelf inserted into center divider slots. +* **Shelf l01 b02 -> Right side panel** | Joint: interlocking | Note: Right tabs of shelf inserted into right side panel slots. diff --git a/eval/tasks/muse-cnc_tv_stand_soundbar_low/harness.ts b/eval/tasks/muse-cnc_tv_stand_soundbar_low/harness.ts new file mode 100644 index 000000000..e7a875507 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_soundbar_low/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_soundbar_low/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_soundbar_low' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_soundbar_low'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_soundbar_low/prompt.md b/eval/tasks/muse-cnc_tv_stand_soundbar_low/prompt.md new file mode 100644 index 000000000..26c231570 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_soundbar_low/prompt.md @@ -0,0 +1,112 @@ +# cnc_tv_stand_soundbar_low (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a low-profile, soundbar-friendly TV stand with a shallow shelf and a generous open front bay, designed for flat-pack assembly. + +## Geometry and Dimensions +Approx. 1500.0 mm × 360.0 mm × 380.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage (supporting a television, soundbar, and media accessories). + +## Structural Features +Top panel; bottom panel; left side panel; right side panel; shallow shelf. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +5 + +## Component Names +- top_panel +- bottom_panel +- left_side_panel +- right_side_panel +- shelf_l01_b01 + +## Adjustable Parameters +- **width**: 1500.0 (1400.0 ~ 1640.0 mm). Defines the overall span of the TV stand. +- **depth**: 360.0 (260.0 ~ 500.0 mm). Determines the footprint and anti-overturning stability. +- **height**: 380.0 (320.0 ~ 460.0 mm). Sets the elevation of the top surface for optimal TV viewing height. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Global material thickness ensuring structural stiffness. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Specific thickness for the top load-bearing panel. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Specific thickness for the base panel. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Controls the width of the mortise and tenon joints for assembly. +- **corner_radius**: 12.0 (0.0 ~ 32.0 mm). Rounds the corners of the top panel for safety and aesthetics. +- **shelf_depth**: 300.0 (200.0 ~ 440.0 mm). Depth of the internal storage shelf, kept shallow to accommodate wiring or specific devices. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the shelf panel to prevent sagging under load. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The primary upper surface of the TV stand. +* **Component Purpose**: Acts as the main load-bearing base for the television. Features mortise slots to receive the vertical side panels. +* **Assembly Direction**: Placed downwards along the -Z axis onto the side panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 2. bottom_panel +The foundational base of the TV stand. +* **Component Purpose**: Provides ground contact and stability, tying the side panels together at the bottom to prevent splaying. +* **Assembly Direction**: Placed upwards along the +Z axis (or acts as the base into which side panels are inserted). +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 3. left_side_panel +The left vertical support structure. +* **Component Purpose**: Transfers the load from the top panel to the bottom panel. Features a "wing" profile, a "low" window cutout, top/bottom tenons, and slots for the shelf. +* **Assembly Direction**: Vertical insertion along the Z axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 4. right_side_panel +The right vertical support structure. +* **Component Purpose**: Transfers the load from the top panel to the bottom panel. Mirrors the left panel with a "wing" profile, a "low" window cutout, top/bottom tenons, and slots for the shelf. +* **Assembly Direction**: Vertical insertion along the Z axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 5. shelf_l01_b01 +The internal horizontal storage surface. +* **Component Purpose**: Provides a dedicated platform for a soundbar or media devices. +* **Assembly Direction**: Inserted horizontally or captured between the side panels during assembly. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **left_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tenons of the left panel inserted into the bottom panel's left slots. +* **right_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tenons of the right panel inserted into the bottom panel's right slots. +* **shelf_l01_b01 -> left_side_panel** | Joint: interlocking | Note: Left end of the shelf inserted into the left panel's horizontal slot. +* **shelf_l01_b01 -> right_side_panel** | Joint: interlocking | Note: Right end of the shelf inserted into the right panel's horizontal slot. +* **top_panel -> left_side_panel** | Joint: interlocking | Note: Top panel slots fitted over the top tenons of the left side panel. +* **top_panel -> right_side_panel** | Joint: interlocking | Note: Top panel slots fitted over the top tenons of the right side panel. diff --git a/eval/tasks/muse-cnc_tv_stand_storage_bench/harness.ts b/eval/tasks/muse-cnc_tv_stand_storage_bench/harness.ts new file mode 100644 index 000000000..1e3fd7fa7 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_storage_bench/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_storage_bench/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_storage_bench' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_storage_bench'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_storage_bench/prompt.md b/eval/tasks/muse-cnc_tv_stand_storage_bench/prompt.md new file mode 100644 index 000000000..4e2e8f880 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_storage_bench/prompt.md @@ -0,0 +1,142 @@ +# cnc_tv_stand_storage_bench (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a storage-bench media stand with two shelf levels and a broad top surface, designed for CNC-machined wood assembly. + +## Geometry and Dimensions +Approx. 1440.0 mm × 420.0 mm × 560.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage, supporting media equipment (e.g., TV) and providing internal shelf storage. + +## Structural Features +Top panel; bottom panel; two side panels (left/right); center divider; four shelf segments. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- top_panel +- bottom_panel +- left_side_panel +- right_side_panel +- center_divider_01 +- shelf_l01_b01 +- shelf_l01_b02 +- shelf_l02_b01 +- shelf_l02_b02 + +## Adjustable Parameters +- **width**: 1440.0 (1340.0 ~ 1580.0 mm). Defines the overall span of the stand. +- **depth**: 420.0 (320.0 ~ 560.0 mm). Defines the footprint depth of the stand. +- **height**: 560.0 (500.0 ~ 640.0 mm). Defines the overall height of the stand. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the thickness of the main vertical support panels. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the thickness of the top panel to ensure adequate load-bearing capacity for media equipment. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the thickness of the bottom base panel. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Defines the width of the tenon tabs used for interlocking the panels. +- **corner_radius**: 8.0 (0.0 ~ 28.0 mm). Defines the rounding radius of the top panel corners for safety and aesthetics. +- **shelf_depth**: 360.0 (260.0 ~ 500.0 mm). Defines the depth of the internal shelves, typically slightly recessed from the main frame. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Defines the thickness of the shelf panels to prevent sagging under load. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The main upper surface of the media stand. +* **Component Purpose**: Provides a broad, continuous surface for placing a TV or other media equipment. Features mortise slots on its underside to receive the tenons of the vertical supports. +* **Assembly Direction**: Placed downwards along the -Z axis onto the vertical supports. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 2. bottom_panel +The base surface of the media stand. +* **Component Purpose**: Provides a structural base and stability for the entire assembly. Features mortise slots to receive the bottom tenons of the vertical supports. +* **Assembly Direction**: Fixed base component, positioned at the bottom (absolute Z = 0). +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 3. left_side_panel +The left vertical support structure. +* **Component Purpose**: Supports the top panel and shelves on the left side. Features tenons on its top and bottom edges, and mortise slots on its inner face for the shelves. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 4. right_side_panel +The right vertical support structure. +* **Component Purpose**: Supports the top panel and shelves on the right side. Features tenons on its top and bottom edges, and mortise slots on its inner face for the shelves. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 5. center_divider_01 +The central vertical support structure. +* **Component Purpose**: Divides the internal space into two bays and provides central load-bearing support for the top panel and shelves to prevent sagging. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 6. shelf_l01_b01 +The lower shelf in the left bay. +* **Component Purpose**: Provides a horizontal storage surface within the first (lower) level of the left bay. +* **Assembly Direction**: Inserted horizontally between the left side panel and the center divider. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 7. shelf_l01_b02 +The lower shelf in the right bay. +* **Component Purpose**: Provides a horizontal storage surface within the first (lower) level of the right bay. +* **Assembly Direction**: Inserted horizontally between the center divider and the right side panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 8. shelf_l02_b01 +The upper shelf in the left bay. +* **Component Purpose**: Provides a horizontal storage surface within the second (upper) level of the left bay. +* **Assembly Direction**: Inserted horizontally between the left side panel and the center divider. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +### 9. shelf_l02_b02 +The upper shelf in the right bay. +* **Component Purpose**: Provides a horizontal storage surface within the second (upper) level of the right bay. +* **Assembly Direction**: Inserted horizontally between the center divider and the right side panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **left_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tenons of the left panel inserted into the bottom panel's left slots. +* **right_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tenons of the right panel inserted into the bottom panel's right slots. +* **center_divider_01 -> bottom_panel** | Joint: interlocking | Note: Bottom tenons of the center divider inserted into the bottom panel's center slots. +* **shelf_l01_b01 -> left_side_panel, center_divider_01** | Joint: interlocking | Note: Level 1, Bay 1 shelf tenons inserted into the corresponding slots of the left panel and center divider. +* **shelf_l01_b02 -> center_divider_01, right_side_panel** | Joint: interlocking | Note: Level 1, Bay 2 shelf tenons inserted into the corresponding slots of the center divider and right panel. +* **shelf_l02_b01 -> left_side_panel, center_divider_01** | Joint: interlocking | Note: Level 2, Bay 1 shelf tenons inserted into the corresponding slots of the left panel and center divider. +* **shelf_l02_b02 -> center_divider_01, right_side_panel** | Joint: interlocking | Note: Level 2, Bay 2 shelf tenons inserted into the corresponding slots of the center divider and right panel. +* **top_panel -> left_side_panel, right_side_panel, center_divider_01** | Joint: interlocking | Note: Top panel slots fit over the top tenons of all three vertical supports. diff --git a/eval/tasks/muse-cnc_tv_stand_three_bay_console/harness.ts b/eval/tasks/muse-cnc_tv_stand_three_bay_console/harness.ts new file mode 100644 index 000000000..9c61d8271 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_three_bay_console/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-cnc_tv_stand_three_bay_console/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'cnc_tv_stand_three_bay_console' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-cnc_tv_stand_three_bay_console'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-cnc_tv_stand_three_bay_console/prompt.md b/eval/tasks/muse-cnc_tv_stand_three_bay_console/prompt.md new file mode 100644 index 000000000..831294359 --- /dev/null +++ b/eval/tasks/muse-cnc_tv_stand_three_bay_console/prompt.md @@ -0,0 +1,118 @@ +# cnc_tv_stand_three_bay_console (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a three-bay media console (TV stand) with two internal dividers and a continuous shelf level, designed for flat-pack CNC-machined assembly. + +## Geometry and Dimensions +Approx. 1680.0 mm × 420.0 mm × 480.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage (supports television and media equipment). + +## Structural Features +Top panel; bottom panel; left side panel; right side panel; two center dividers; three shelf segments. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- top_panel +- bottom_panel +- left_side_panel +- right_side_panel +- center_divider_01 +- center_divider_02 +- shelf_l01_b01 +- shelf_l01_b02 +- shelf_l01_b03 + +## Adjustable Parameters +- **width**: 1680.0 (1580.0 ~ 1820.0 mm). Defines the overall span of the media console. +- **depth**: 420.0 (320.0 ~ 560.0 mm). Determines the storage capacity and footprint. +- **height**: 480.0 (420.0 ~ 560.0 mm). Sets the vertical elevation of the console. +- **thickness**: 18.0 (12.0 ~ 26.0 mm). Standard material thickness for vertical structural stability. +- **top_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the top load-bearing surface to prevent sagging under TV weight. +- **bottom_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the base panel. +- **tab_width**: 18.0 (100.0 ~ 158.0 mm). Controls the width of the tenons/tabs for assembly joints. +- **corner_radius**: 12.0 (0.0 ~ 32.0 mm). Defines the rounding of the top panel corners for safety and aesthetics. +- **shelf_depth**: 372.0 (272.0 ~ 512.0 mm). Depth of the internal shelving. +- **shelf_thickness**: 18.0 (12.0 ~ 26.0 mm). Thickness of the shelf panels to support media devices. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. top_panel +The main upper load-bearing surface. +* **Component Purpose**: Supports the television and encloses the top of the console. Provides mortise slots to locate and secure the side panels and dividers. +* **Assembly Direction**: Placed downwards along the -Z axis onto the vertical supports. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 2. bottom_panel +The base structural surface. +* **Component Purpose**: Acts as the foundational base, providing mortise slots for all vertical panels to ensure structural rigidity. +* **Assembly Direction**: Fixed base component, positioned at the bottom of the assembly. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 3~4. left_side_panel & right_side_panel +The outer vertical supports. +* **Component Purpose**: Encloses the sides of the console and transfers the load from the top panel to the bottom panel. Features tabs (tenons) on the top/bottom and slots for the outer shelf segments. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 5~6. center_divider_01 & center_divider_02 +The internal vertical supports. +* **Component Purpose**: Divides the console into three distinct bays, supports the top panel mid-span to prevent sagging, and provides bilateral slots for the inner shelf segments. +* **Assembly Direction**: Inserted vertically between the top and bottom panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +### 7~9. shelf_l01_b01, shelf_l01_b02, & shelf_l01_b03 +The horizontal storage dividers. +* **Component Purpose**: Provides internal storage levels within each of the three bays for media equipment. +* **Assembly Direction**: Inserted horizontally into the slots of the vertical panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **left_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's far-left slots. +* **right_side_panel -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's far-right slots. +* **center_divider_01 -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's inner-left slots. +* **center_divider_02 -> bottom_panel** | Joint: interlocking | Note: Bottom tabs inserted into the bottom panel's inner-right slots. +* **top_panel -> All Vertical Panels** | Joint: interlocking | Note: Top panel slots fit over the top tabs of the side panels and center dividers. +* **shelf_l01_b01 -> left_side_panel & center_divider_01** | Joint: interlocking | Note: Left bay shelf tabs inserted into the adjacent vertical supports. +* **shelf_l01_b02 -> center_divider_01 & center_divider_02** | Joint: interlocking | Note: Center bay shelf tabs inserted into the two center dividers. +* **shelf_l01_b03 -> center_divider_02 & right_side_panel** | Joint: interlocking | Note: Right bay shelf tabs inserted into the adjacent vertical supports. diff --git a/eval/tasks/muse-coat_rack/harness.ts b/eval/tasks/muse-coat_rack/harness.ts new file mode 100644 index 000000000..893deb993 --- /dev/null +++ b/eval/tasks/muse-coat_rack/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-coat_rack/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'coat_rack' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-coat_rack'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-coat_rack/prompt.md b/eval/tasks/muse-coat_rack/prompt.md new file mode 100644 index 000000000..db919deb8 --- /dev/null +++ b/eval/tasks/muse-coat_rack/prompt.md @@ -0,0 +1,136 @@ +# coat_rack (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a wooden coat rack featuring upper and lower hanging cross rails and a slatted bottom storage shelf, designed for dowel-based assembly. + +## Geometry and Dimensions +Approx. 530.0 mm × 360.0 mm × 1700.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Load-bearing storage for hanging clothing and placing accessories on the bottom shelf. + +## Structural Features +Four vertical uprights; top and bottom side rails; upper and lower cross rails (acting as hanging pegs); shelf rails; slatted bottom shelf. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +23 + +## Component Names +- right_front_upright +- right_back_upright +- left_front_upright +- left_back_upright +- right_bottom_rail +- left_bottom_rail +- right_top_rail +- left_top_rail +- upper_front_cross +- upper_back_cross +- lower_front_cross +- lower_back_cross +- left_shelf_rail +- right_shelf_rail +- shelf_slat_01 +- shelf_slat_02 +- shelf_slat_03 +- shelf_slat_04 +- shelf_slat_05 +- shelf_slat_06 +- shelf_slat_07 +- shelf_slat_08 +- shelf_slat_09 + +## Adjustable Parameters +- **rack_width**: 500.0 (350.0 ~ 700.0 mm). Defines the overall width of the rack, ensuring adequate hanging space without compromising structural stability. +- **rack_depth**: 350.0 (250.0 ~ 500.0 mm). Defines the depth footprint, balancing the anti-overturning base with spatial constraints. +- **rack_height**: 1700.0 (1400.0 ~ 2000.0 mm). Defines the total height of the uprights to accommodate long coats. +- **peg_height_upper**: 1500.0 (1300.0 ~ 1800.0 mm). Sets the Z-height for the upper hanging cross rails for standard adult reach. +- **peg_height_lower**: 1200.0 (900.0 ~ 1400.0 mm). Sets the Z-height for the lower hanging cross rails for shorter garments or accessible reach. +- **shelf_height**: 150.0 (80.0 ~ 300.0 mm). Sets the Z-height for the bottom storage shelf, keeping items off the floor. +- **board_thickness**: 10.0 (6.0 ~ 16.0 mm). Thickness of the timber boards used, ensuring sufficient material for blind dowel holes. +- **board_width**: 30.0 (20.0 ~ 45.0 mm). Width of the timber boards used, providing structural stiffness to the frame. +- **slat_gap**: 7.0 (3.0 ~ 12.0 mm). Spacing between adjacent shelf slats, dynamically determining the total number of slats based on rack depth. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Radius of the dowel holes for assembly alignment and connection. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Depth of the blind dowel holes to ensure they do not pierce through the outer faces of the boards. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~4. Uprights (Right Front, Right Back, Left Front, Left Back) +The primary vertical support structures of the coat rack. +* **Component Purpose**: Vertical support. Transfers the load to the ground and provides dowel hole interfaces on the inner faces for side rails, shelf rails, and cross rails. +* **Assembly Direction**: Vertical base components, positioned upright along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features blind holes on the X and Y inner faces. + +### 5~8. Side Rails (Right Bottom, Left Bottom, Right Top, Left Top) +The horizontal depth-wise bracing components. +* **Component Purpose**: Connects the front and back uprights to ensure structural stability and prevent racking in the Y-Z plane. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features dowel holes at both ends mating with the uprights. + +### 9~12. Cross Rails (Upper Front, Upper Back, Lower Front, Lower Back) +The horizontal width-wise bracing and functional hanging components. +* **Component Purpose**: Connects the left and right uprights to prevent racking in the X-Z plane, while simultaneously acting as the primary hanging pegs for garments. +* **Assembly Direction**: Inserted horizontally along the X-axis between the left and right uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features dowel holes at both ends mating with the uprights. + +### 13~14. Shelf Rails (Left, Right) +The horizontal supports for the bottom shelf. +* **Component Purpose**: Connects the front and back uprights at the designated shelf height and provides vertical dowel holes on the top face to mount the shelf slats. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). + +### 15~23. Shelf Slats (01 to 09) +The surface components of the bottom storage shelf. +* **Component Purpose**: Spans across the left and right shelf rails to form a slatted platform for storing shoes or bags. +* **Assembly Direction**: Placed downwards along the -Z axis onto the shelf rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical dowel holes on the bottom face mating with the shelf rails. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 23-component model: + +* **Right Bottom Rail -> Right Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the bottom base. +* **Left Bottom Rail -> Left Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the bottom base. +* **Right Top Rail -> Right Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the top. +* **Left Top Rail -> Left Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the top. +* **Upper Front Cross Rail -> Left & Right Front Uprights** | Joint: Dowel Joint | Note: Connects left and right uprights at the upper peg height. +* **Upper Back Cross Rail -> Left & Right Back Uprights** | Joint: Dowel Joint | Note: Connects left and right uprights at the upper peg height. +* **Lower Front Cross Rail -> Left & Right Front Uprights** | Joint: Dowel Joint | Note: Connects left and right uprights at the lower peg height. +* **Lower Back Cross Rail -> Left & Right Back Uprights** | Joint: Dowel Joint | Note: Connects left and right uprights at the lower peg height. +* **Left Shelf Rail -> Left Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the shelf height. +* **Right Shelf Rail -> Right Front & Back Uprights** | Joint: Dowel Joint | Note: Connects front and back uprights at the shelf height. +* **Shelf Slats (01-09) -> Left & Right Shelf Rails** | Joint: Dowel Joint | Note: Slats are mounted on top of the shelf rails via vertical dowel holes. diff --git a/eval/tasks/muse-comb_bonded_backing_timber/harness.ts b/eval/tasks/muse-comb_bonded_backing_timber/harness.ts new file mode 100644 index 000000000..19513123a --- /dev/null +++ b/eval/tasks/muse-comb_bonded_backing_timber/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-comb_bonded_backing_timber/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'comb_bonded_backing_timber' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-comb_bonded_backing_timber'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-comb_bonded_backing_timber/prompt.md b/eval/tasks/muse-comb_bonded_backing_timber/prompt.md new file mode 100644 index 000000000..a0284dbbd --- /dev/null +++ b/eval/tasks/muse-comb_bonded_backing_timber/prompt.md @@ -0,0 +1,83 @@ +# comb_bonded_backing_timber (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a two-piece timber comb featuring a fine-tooth strip bonded to an ergonomic backing plate with an extended handle for personal grooming. + +## Geometry and Dimensions +Approx. 213.0 mm × 58.0 mm × 4.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Handheld personal care and grooming tool. Subject to bending forces and cantilever loads on the teeth during use. + +## Structural Features +Tooth strip with embedded teeth; ergonomic backing plate with an extended handle. + +## Special Requirements +Keep assembly split unchanged. Ensure mating faces (the top edge of the tooth strip and the bottom edge of the backing plate) are perfectly flush to maximize the surface area for bonding. + +## Planned Component Quantity +2 + +## Component Names +- tooth_strip +- ergonomic_back_handle + +## Adjustable Parameters +- **teeth_count**: 18 (10 ~ 40). Determines the total number of combing teeth; affects the overall length of the functional area. +- **tooth_width**: 3.4 (2.0 ~ 6.0 mm). Controls the thickness of individual teeth; must be large enough to prevent fracture during CNC milling and usage. +- **gap**: 2.8 (1.5 ~ 5.0 mm). Defines the spacing between teeth; constrained by the minimum tool radius of the CNC end mill. +- **tooth_length**: 28.0 (15.0 ~ 50.0 mm). Determines the penetration depth of the comb; longer teeth require careful feed rates during machining to avoid snapping. +- **strip_depth**: 8.0 (5.0 ~ 15.0 mm). The height of the continuous root bed that anchors the teeth before mating with the handle. +- **thickness**: 4.0 (3.0 ~ 10.0 mm). The global thickness of the timber stock used for both components. +- **back_depth**: 22.0 (15.0 ~ 40.0 mm). Defines the vertical height of the ergonomic backing plate, providing structural rigidity to the spine. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. tooth_strip +The functional grooming interface of the comb. +* **Component Purpose**: Provides the array of teeth for combing, embedded into a continuous root bed to distribute stress and prevent individual tooth failure. +* **Assembly Direction**: Coplanar, mates along its top edge (Y=8.0) to the backing plate. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). The top face of the root bed is glued to the bottom face of the ergonomic back handle. + +### 2. ergonomic_back_handle +The structural support and user interface of the comb. +* **Component Purpose**: Acts as the rigid spine to support the tooth strip and extends horizontally past the teeth to form an ergonomic handle for the user. +* **Assembly Direction**: Coplanar, mates along its bottom edge (Y=8.0) to the tooth strip. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). The bottom face of the backing plate is glued to the top face of the tooth strip. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **tooth_strip -> ergonomic_back_handle** | Joint: Bonding (Glue) | Note: The top edge of the tooth strip's root bed is permanently bonded to the bottom edge of the ergonomic backing plate. diff --git a/eval/tasks/muse-comb_guard_sleeve_pla/harness.ts b/eval/tasks/muse-comb_guard_sleeve_pla/harness.ts new file mode 100644 index 000000000..dc5be4a20 --- /dev/null +++ b/eval/tasks/muse-comb_guard_sleeve_pla/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-comb_guard_sleeve_pla/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'comb_guard_sleeve_pla' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-comb_guard_sleeve_pla'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-comb_guard_sleeve_pla/prompt.md b/eval/tasks/muse-comb_guard_sleeve_pla/prompt.md new file mode 100644 index 000000000..a08b13b2f --- /dev/null +++ b/eval/tasks/muse-comb_guard_sleeve_pla/prompt.md @@ -0,0 +1,83 @@ +# comb_guard_sleeve_pla (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Design a handheld comb with an integrated tapered handle and a matching protective guard sleeve for the teeth, optimized for 3D printing and portable storage. + +## Geometry and Dimensions +Approx. 180.0 mm × 31.0 mm × 8.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Snap-fit + +## Mechanical Condition +Handheld personal care and grooming, with a protective cover for safe storage and transport to prevent tooth breakage. + +## Structural Features +Comb with integrated handle and teeth; hollow protective guard sleeve. + +## Special Requirements +Keep assembly split unchanged. The guard cavity must maintain a zero-overlap tolerance with the teeth to ensure proper friction/snap-fit retention. + +## Planned Component Quantity +2 + +## Component Names +- Comb with handle +- Tooth guard sleeve + +## Adjustable Parameters +- **teeth_count**: 18 (10 ~ 50). Determines the density and overall functional width of the comb head. +- **tooth_width**: 3.2 (1.5 ~ 5.0 mm). Affects the structural strength and flexibility of individual teeth. +- **gap**: 2.4 (1.0 ~ 5.0 mm). Controls the spacing between teeth for optimal hair passage. +- **tooth_length**: 28.0 (15.0 ~ 50.0 mm). Determines the combing depth and the required depth of the guard sleeve. +- **spine_depth**: 10.0 (5.0 ~ 20.0 mm). Provides structural rigidity to the comb head to withstand bending moments during use. +- **thickness**: 4.0 (2.0 ~ 8.0 mm). Overall thickness of the comb, ensuring sufficient stiffness for FDM printing. +- **handle_length**: 70.0 (50.0 ~ 120.0 mm). Provides an ergonomic grip length for handheld operation. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Comb with handle +The main functional grooming tool. +* **Component Purpose**: Used for combing hair; features a tapered handle for ergonomic grip and an array of rounded teeth embedded into a rigid spine. +* **Assembly Direction**: Inserted downwards along the -Y axis into the tooth guard sleeve. +* **Connection & Kinematics**: Snap-fit (Fully constrained in all directions (locking)). + +### 2. Tooth guard sleeve +The protective cover for the comb head. +* **Component Purpose**: Protects the comb teeth from mechanical damage and prevents snagging during transport. Features a custom cavity that exactly matches the teeth volume. +* **Assembly Direction**: Receives the comb from the +Y direction through the top mouth opening. +* **Connection & Kinematics**: Snap-fit (Fully constrained in all directions (locking)). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **Comb with handle -> Tooth guard sleeve** | Joint: Snap-fit | Note: Comb teeth slide into the guard sleeve's internal cavity through the top mouth opening, locking in place via friction/snap-fit. diff --git a/eval/tasks/muse-comb_sheet_metal_wallet/harness.ts b/eval/tasks/muse-comb_sheet_metal_wallet/harness.ts new file mode 100644 index 000000000..0380608f1 --- /dev/null +++ b/eval/tasks/muse-comb_sheet_metal_wallet/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-comb_sheet_metal_wallet/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'comb_sheet_metal_wallet' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-comb_sheet_metal_wallet'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-comb_sheet_metal_wallet/prompt.md b/eval/tasks/muse-comb_sheet_metal_wallet/prompt.md new file mode 100644 index 000000000..e4fc3c1d9 --- /dev/null +++ b/eval/tasks/muse-comb_sheet_metal_wallet/prompt.md @@ -0,0 +1,72 @@ +# comb_sheet_metal_wallet (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a multi-functional, credit-card-sized everyday carry (EDC) wallet comb with integrated utility features including a bottle opener and lanyard hole. + +## Geometry and Dimensions +Approx. 85.0 mm × 54.0 mm × 1.2 mm. + +## Material +Sheet Metal + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Everyday carry (EDC) in a standard wallet card slot; grooming; light utility tasks such as opening bottles. + +## Structural Features +Credit-card footprint plate; comb teeth array on the long edge; lanyard hole; bottle opener cutout. + +## Special Requirements +Must maintain standard credit card outer dimensions (85x54mm) to fit in a wallet. Outer corners must be filleted (4.0mm radius) to prevent snagging on fabric or leather. + +## Planned Component Quantity +1 + +## Component Names +- wallet_comb_plate + +## Adjustable Parameters +- **thickness**: 1.2 (0.8 ~ 2.0 mm). Determines the rigidity of the sheet metal and ensures it fits comfortably within a standard wallet slot without bending. +- **teeth_count**: 26 (15 ~ 40). Defines the density of the comb for different hair types. +- **tooth_width**: 1.4 (1.0 ~ 3.0 mm). Balances individual tooth strength against grooming comfort. +- **gap**: 1.5 (1.0 ~ 3.0 mm). Controls the spacing between teeth for hair passage and laser cutting clearance. +- **slit_length**: 18.0 (10.0 ~ 25.0 mm). Determines the effective combing depth along the edge of the plate. +- **slit_escape**: 4.0 (2.0 ~ 8.0 mm). Provides a stress-relief radius at the root of the comb teeth to prevent fatigue fracture. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. wallet_comb_plate +The main and only body of the EDC tool. +* **Component Purpose**: Serves as a monolithic multi-tool, integrating comb teeth via edge slits and utility cutouts (bottle opener, lanyard hole) into a single flat profile. +* **Assembly Direction**: None (Standalone part). +* **Connection & Kinematics**: None (Fully constrained as a single solid body). + +--- + +## Component Assembly Graph (Textual) +* **wallet_comb_plate -> Standalone** | Joint: None | Note: Single-piece monolithic design; no assembly required. diff --git a/eval/tasks/muse-comb_snap_handle_abs/harness.ts b/eval/tasks/muse-comb_snap_handle_abs/harness.ts new file mode 100644 index 000000000..f6c903d84 --- /dev/null +++ b/eval/tasks/muse-comb_snap_handle_abs/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-comb_snap_handle_abs/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'comb_snap_handle_abs' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-comb_snap_handle_abs'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-comb_snap_handle_abs/prompt.md b/eval/tasks/muse-comb_snap_handle_abs/prompt.md new file mode 100644 index 000000000..1ced36504 --- /dev/null +++ b/eval/tasks/muse-comb_snap_handle_abs/prompt.md @@ -0,0 +1,84 @@ +# comb_snap_handle_abs (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a two-part modular comb with a detachable handle, designed for rapid prototyping and assembly via an integrated locking mechanism. + +## Geometry and Dimensions +Approx. 205.0 mm × 38.0 mm × 6.0 mm. + +## Material +ABS + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Snap-fit + +## Mechanical Condition +Handheld personal grooming tool, subject to light cantilever bending loads and shear forces during hair combing. + +## Structural Features +Comb head with integrated teeth and spine; tapered ergonomic handle. + +## Special Requirements +Keep assembly split unchanged. Ensure the retention bumps and recesses maintain appropriate tolerances to allow for elastic deformation during the snap-fit insertion. + +## Planned Component Quantity +2 + +## Component Names +- snap_comb_head +- snap_handle_frame + +## Adjustable Parameters +- **teeth_count**: 20 (10 ~ 40). Determines the combing density and the overall width of the comb head. +- **tooth_width**: 3.0 (1.5 ~ 5.0 mm). Ensures individual tooth strength against bending and snapping. +- **gap**: 2.3 (1.0 ~ 5.0 mm). Controls the spacing between teeth for optimal hair passage. +- **tooth_length**: 28.0 (15.0 ~ 50.0 mm). Defines the effective combing depth. +- **spine_depth**: 10.0 (5.0 ~ 20.0 mm). Provides structural rigidity to the comb head base to prevent bowing. +- **thickness**: 4.0 (2.0 ~ 8.0 mm). Overall thickness of the comb and handle, balancing stiffness and material usage. +- **handle_len**: 88.0 (50.0 ~ 150.0 mm). Ergonomic length for user grip and leverage. +- **tenon_len**: 18.0 (10.0 ~ 40.0 mm). Insertion depth for the snap-fit joint to ensure mechanical stability and prevent wobble. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. snap_comb_head +The functional grooming interface of the assembly. +* **Component Purpose**: Acts as the primary tool for combing, featuring 20 rounded teeth and a locking tenon with top and bottom retention bumps for secure attachment. +* **Assembly Direction**: Inserted horizontally along the +X axis into the handle frame. +* **Connection & Kinematics**: Snap-fit (Fully constrained in all directions (locking)). The extended tenon features cylindrical bumps that act as the male locking mechanism. + +### 2. snap_handle_frame +The structural grip entity of the assembly. +* **Component Purpose**: Provides an ergonomic, tapered grip for the user. It contains the female mating geometry to securely anchor the comb head. +* **Assembly Direction**: Receives the comb head along the -X axis (fixed base relative to the head's insertion). +* **Connection & Kinematics**: Snap-fit (Fully constrained in all directions (locking)). Features an exact-size mortise with internal cylindrical recesses generated via boolean cut to capture the comb head's retention bumps. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **snap_comb_head -> snap_handle_frame** | Joint: Snap-fit | Note: Comb head tenon with retention bumps inserts into the handle's mortise and locks into the internal recesses. diff --git a/eval/tasks/muse-comb_wide_detangler_pla/harness.ts b/eval/tasks/muse-comb_wide_detangler_pla/harness.ts new file mode 100644 index 000000000..5de2dcd43 --- /dev/null +++ b/eval/tasks/muse-comb_wide_detangler_pla/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-comb_wide_detangler_pla/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'comb_wide_detangler_pla' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-comb_wide_detangler_pla'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-comb_wide_detangler_pla/prompt.md b/eval/tasks/muse-comb_wide_detangler_pla/prompt.md new file mode 100644 index 000000000..2295f839e --- /dev/null +++ b/eval/tasks/muse-comb_wide_detangler_pla/prompt.md @@ -0,0 +1,74 @@ +# comb_wide_detangler_pla (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a wide-tooth detangling comb designed for FDM 3D printing, featuring a reinforced spine and rounded teeth optimized for hair care. + +## Geometry and Dimensions +Approx. 119.6 mm × 47.0 mm × 4.5 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Handheld personal care item; teeth experience cantilever bending stress during hair detangling. + +## Structural Features +Main spine/handle; 12 wide teeth with widened roots for structural reinforcement. + +## Special Requirements +Keep as a single fused solid body. Ensure teeth roots are deeply embedded and fused into the spine to prevent snapping along layer lines. + +## Planned Component Quantity +1 + +## Component Names +- wide_detangler_body + +## Adjustable Parameters +- **teeth_count**: 12. Determines the total number of detangling teeth. +- **tooth_width**: 4.6 mm. Defines the width of each tooth to ensure structural rigidity. +- **gap**: 4.4 mm. Sets the spacing between teeth, optimized for wide detangling without snagging. +- **tooth_length**: 34.0 mm. Determines the reach of the comb through hair. +- **spine_depth**: 13.0 mm. Provides the main structural backbone and gripping area. +- **thickness**: 4.5 mm. Overall Z-axis thickness ensuring printability and bending resistance. +- **end_margin**: 8.0 mm. Adds extra material at the left and right ends of the spine for handling and strength. +- **first_last_length_scale**: 0.85. Shortens the outermost teeth to create a tapered, ergonomic profile and reduce edge snagging. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main spine profile. +2. Generate the array of teeth with widened roots and scale the outer teeth. +3. Fuse all teeth and the spine into a single continuous solid. + +--- + +### 1. wide_detangler_body +The monolithic structure of the comb. +* **Component Purpose**: Acts as both the structural handle (spine) and the functional interface (teeth) for detangling hair. +* **Assembly Direction**: N/A (Single component). +* **Connection & Kinematics**: None (Single Body). + +--- + +## Component Assembly Graph (Textual) +wide_detangler_body -> None | Joint: None | Note: Single monolithic part; no assembly required. diff --git a/eval/tasks/muse-fluted_pen_holder/harness.ts b/eval/tasks/muse-fluted_pen_holder/harness.ts new file mode 100644 index 000000000..9d2bbeb50 --- /dev/null +++ b/eval/tasks/muse-fluted_pen_holder/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-fluted_pen_holder/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'fluted_pen_holder' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-fluted_pen_holder'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-fluted_pen_holder/prompt.md b/eval/tasks/muse-fluted_pen_holder/prompt.md new file mode 100644 index 000000000..3f01548a7 --- /dev/null +++ b/eval/tasks/muse-fluted_pen_holder/prompt.md @@ -0,0 +1,81 @@ +# fluted_pen_holder (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a desktop pen holder featuring a decorative fluted exterior and a hollow internal cavity for organizing pens and stationery. + +## Geometry and Dimensions +Approx. 74.0 mm × 74.0 mm × 96.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Desktop storage, holding lightweight stationery items. + +## Structural Features +Main hollow cup body; rounded lower base; exterior vertical fluted ribs; filleted top opening. + +## Special Requirements +Keep exported STEP as a single closed solid. + +## Planned Component Quantity +1 + +## Component Names +- fluted_pen_holder_body + +## Adjustable Parameters +- **body_width**: 74.0 (50.0 ~ 110.0 mm). Determines the overall width of the pen holder. +- **body_depth**: 74.0 (50.0 ~ 110.0 mm). Determines the overall depth of the pen holder. +- **total_height**: 96.0 (70.0 ~ 150.0 mm). Controls the vertical capacity for holding pens. +- **base_height**: 18.0 (10.0 ~ 35.0 mm). Defines the height of the lower base section. +- **base_inset**: 1.2 (0.0 ~ 4.0 mm). Controls the step-in distance of the base relative to the upper body, creating a slight overhang. +- **wall_thickness**: 3.8 (2.0 ~ 8.0 mm). Ensures structural integrity and stiffness of the cup walls. +- **floor_thickness**: 5.0 (3.0 ~ 10.0 mm). Provides a solid bottom to prevent pens from piercing through and adds weight for anti-tipping stability. +- **upper_corner_radius**: 7.0 (3.0 ~ 14.0 mm). Softens the vertical edges of the main upper body. +- **base_corner_radius**: 10.0 (4.0 ~ 16.0 mm). Softens the vertical edges of the base. +- **top_edge_radius**: 1.2 (0.4 ~ 3.0 mm). Softens the mouth opening for ergonomic safety and aesthetics. +- **rib_width**: 1.4 (0.8 ~ 3.0 mm). Defines the thickness of the exterior decorative ribs. +- **rib_depth**: 1.2 (0.4 ~ 2.4 mm). Defines how far the ribs protrude from the main body. +- **rib_pitch**: 4.0 (2.5 ~ 8.0 mm). Controls the spacing and density of the fluted ribs along the exterior faces. +- **rib_margin**: 8.0 (4.0 ~ 14.0 mm). Sets the blank space at the corners where ribs are not placed to avoid geometric interference at the fillets. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the outer body (base and upper cup). +2. Cut the inner cavity to hollow out the holder. +3. Generate and fuse the exterior vertical ribs. +4. Apply fillets to the top edge of the mouth opening. + +--- + +### 1. fluted_pen_holder_body +The central and sole entity of the model. +* **Component Purpose**: Acts as the main storage container, providing a stable base and a decorative fluted exterior for desktop organization. +* **Assembly Direction**: N/A (Standalone object). +* **Connection & Kinematics**: Not applicable (Single monolithic component). + +--- + +## Component Assembly Graph (Textual) +fluted_pen_holder_body -> Standalone | Joint: None | Note: Single monolithic component; no assembly required. diff --git a/eval/tasks/muse-handle_comb/harness.ts b/eval/tasks/muse-handle_comb/harness.ts new file mode 100644 index 000000000..bfa0d62b1 --- /dev/null +++ b/eval/tasks/muse-handle_comb/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-handle_comb/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'handle_comb' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-handle_comb'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-handle_comb/prompt.md b/eval/tasks/muse-handle_comb/prompt.md new file mode 100644 index 000000000..b6a8f8401 --- /dev/null +++ b/eval/tasks/muse-handle_comb/prompt.md @@ -0,0 +1,74 @@ +# handle_comb (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a handheld grooming comb featuring an ergonomically lofted handle and evenly spaced teeth for hair detangling and styling. + +## Geometry and Dimensions +Approx. 3.0 mm × 26.0 mm × 137.0 mm. + +## Material +Resin + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Handheld manual grooming; the teeth experience mild cantilever bending forces during use, while the handle requires ergonomic grip stability. + +## Structural Features +Ergonomic lofted handle; solid structural spine; evenly spaced middle teeth; reinforced start and end teeth. + +## Special Requirements +The lofted handle must transition smoothly into the spine without non-manifold edges. The surface finish must be smooth to prevent hair snagging, making SLA printing the optimal choice. + +## Planned Component Quantity +1 + +## Component Names +- comb_body + +## Adjustable Parameters +- **comb_width**: 3 (1.5 ~ 8.0 mm). Controls the overall thickness and baseline rigidity of the comb. +- **handle_length**: 80 (40.0 ~ 120.0 mm). Determines the length of the ergonomic grip area to accommodate different hand sizes. +- **teeth_count**: 20 (8 ~ 40). Defines the density and the total functional length of the combing section. +- **teeth_gap_distance**: 3 (1.0 ~ 8.0 mm). Sets the spacing between teeth, dictating whether it functions as a fine-tooth or wide-tooth comb. +- **teeth_height**: 1 (0.5 ~ 4.0 mm). Controls the individual thickness of the middle teeth, balancing flexibility and strength. +- **teeth_length**: 20 (8.0 ~ 40.0 mm). Determines how deep the comb can penetrate hair layers. +- **spine_length**: 6 (3.0 ~ 16.0 mm). Provides the structural backing required to support the teeth and prevent snapping under bending loads. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent, fully fused geometric body. +2. The exported STEP must remain a closed solid without internal intersecting faces. + +**Global Modeling Steps** +1. Generate the lofted handle using the scaled profile sections. +2. Extrude the main spine along the Z-axis. +3. Generate the array of teeth, applying specific heights to the start and end teeth for reinforcement. +4. Boolean union all solids (handle, spine, teeth) and merge coplanar faces to form a single monolithic body. + +--- + +### 1. comb_body +The single monolithic entity representing the entire comb. +* **Component Purpose**: Acts as both the ergonomic grip interface (handle) and the functional grooming interface (teeth and spine). +* **Assembly Direction**: N/A (Single independent component). +* **Connection & Kinematics**: Not applicable (Monolithic body with 0 Degrees of Freedom internally). + +--- + +## Component Assembly Graph (Textual) +* **comb_body -> Self** | Joint: None | Note: Fused monolithic structure generated via boolean union of handle, spine, and teeth. diff --git a/eval/tasks/muse-laptop_stand_1/harness.ts b/eval/tasks/muse-laptop_stand_1/harness.ts new file mode 100644 index 000000000..c3526aea1 --- /dev/null +++ b/eval/tasks/muse-laptop_stand_1/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-laptop_stand_1/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'laptop_stand_1' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-laptop_stand_1'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-laptop_stand_1/prompt.md b/eval/tasks/muse-laptop_stand_1/prompt.md new file mode 100644 index 000000000..4444bb66f --- /dev/null +++ b/eval/tasks/muse-laptop_stand_1/prompt.md @@ -0,0 +1,88 @@ +# laptop_stand_1 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a modular, interlocking laptop stand designed to elevate the device at an ergonomic viewing angle while providing structural support and heat dissipation space. + +## Geometry and Dimensions +Approx. 30.0 mm × 200.0 mm × 85.0 mm. + +## Material +ABS + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Static load-bearing support for a laptop computer, subject to mild thermal output from the device. + +## Structural Features +Main cradle body; upper support insert; lower support insert. + +## Special Requirements +Keep assembly split unchanged. Maintain the 0.2 mm insert clearance to ensure proper fitment of the printed interlocking parts. + +## Planned Component Quantity +3 + +## Component Names +- upper_support_insert +- lower_support_insert +- main_cradle_body + +## Adjustable Parameters +- **support_angle_deg**: 20 (5.0 ~ 35.0). Controls the ergonomic tilt angle of the laptop support surface. +- **holder_height**: 15 (5.0 ~ 40.0 mm). Determines the height of the front retaining lip to prevent the laptop from sliding off. +- **holder_width**: 10 (4.0 ~ 30.0 mm). Sets the depth/thickness of the front retaining lip for adequate edge grip. +- **support_thickness**: 30 (10.0 ~ 60.0 mm). Controls the overall extrusion width of the stand, directly affecting its footprint and lateral stability. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. upper_support_insert +The top extension module of the stand. +* **Component Purpose**: Acts as the upper contact point for the laptop, interlocking with the main body to extend the support surface. +* **Assembly Direction**: Inserted along the section plane normal into the main cradle body. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a lofted tenon that fits into the corresponding mortise of the main body. + +### 2. lower_support_insert +The bottom extension and retaining module of the stand. +* **Component Purpose**: Provides the front retaining lip (`holder_height` and `holder_width`) to secure the lower edge of the laptop, preventing it from sliding down the angled slope. +* **Assembly Direction**: Inserted along the section plane normal into the main cradle body. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a lofted tenon that fits into the corresponding mortise of the main body. + +### 3. main_cradle_body +The central structural hub of the stand. +* **Component Purpose**: Bridges the upper and lower inserts, bearing the primary weight of the laptop and transferring the load to the resting surface. +* **Assembly Direction**: Fixed base component. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Contains boolean-cut sockets (mortises) with a 0.2 mm clearance to receive the tenons from the upper and lower inserts. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 3-component model: + +* **upper_support_insert -> main_cradle_body** | Joint: interlocking | Note: Insert's lofted tenon is inserted into the upper socket of the main cradle body. +* **lower_support_insert -> main_cradle_body** | Joint: interlocking | Note: Insert's lofted tenon is inserted into the lower socket of the main cradle body. diff --git a/eval/tasks/muse-laptop_stand_2/harness.ts b/eval/tasks/muse-laptop_stand_2/harness.ts new file mode 100644 index 000000000..3f6c5eb2f --- /dev/null +++ b/eval/tasks/muse-laptop_stand_2/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-laptop_stand_2/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'laptop_stand_2' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-laptop_stand_2'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-laptop_stand_2/prompt.md b/eval/tasks/muse-laptop_stand_2/prompt.md new file mode 100644 index 000000000..c04393064 --- /dev/null +++ b/eval/tasks/muse-laptop_stand_2/prompt.md @@ -0,0 +1,105 @@ +# laptop_stand_2 (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a wide, dual-frame laptop or equipment stand designed for stable desktop support and ergonomic viewing angles. + +## Geometry and Dimensions +Approx. 800.0 mm × 800.0 mm × 347.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Static load-bearing support for laptops or similar desktop equipment. + +## Structural Features +Two side frames (front and rear profiles); three cross-support rods (one upper, two base). + +## Special Requirements +Keep assembly split unchanged. Ensure rod clearances are maintained for proper assembly. + +## Planned Component Quantity +5 + +## Component Names +- front_side_frame +- rear_side_frame +- upper_support_rod +- left_base_rod +- right_base_rod + +## Adjustable Parameters +- **panel_thickness**: 40 (20.0 ~ 80.0 mm). Determines the structural rigidity of the side frames and the depth of the rod insertion. +- **rod_radius**: 15 (8.0 ~ 30.0 mm). Controls the thickness of the cross-support rods; lower limit ensures load-bearing stiffness, upper limit prevents interference with the frame profile. +- **rod_span**: 800 (500.0 ~ 1200.0 mm). Defines the overall width of the stand and the distance between the two side frames. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. front_side_frame +The primary side support profile of the stand. +* **Component Purpose**: Acts as the main structural side bracket, providing the angled resting surface for the equipment and housing the insertion holes for the cross rods. +* **Assembly Direction**: Fixed base component, positioned at absolute Y = 0. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features three circular cutouts with a defined `rod_clearance` to accept the support rods. + +### 2. rear_side_frame +The secondary side support profile of the stand. +* **Component Purpose**: Mirrors the front frame to provide parallel support on the opposite side, ensuring lateral stability. +* **Assembly Direction**: Positioned parallel to the front frame, offset along the -Y axis by `rod_span`. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features three circular cutouts identical to the front frame to accept the opposite ends of the support rods. + +### 3. upper_support_rod +The top horizontal connecting cylinder. +* **Component Purpose**: Connects the upper sections of the two side frames, providing structural rigidity and acting as a backstop or upper resting point for the equipment. +* **Assembly Direction**: Inserted horizontally along the Y axis between the frames at support point (40, 0, 295). +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Inserted into the upper circular cutouts of both side frames. + +### 4. left_base_rod +The front/left lower horizontal connecting cylinder. +* **Component Purpose**: Connects the lower front sections of the two side frames, preventing the frames from splaying and providing base stability. +* **Assembly Direction**: Inserted horizontally along the Y axis between the frames at support point (40, 0, 45). +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Inserted into the lower-front circular cutouts of both side frames. + +### 5. right_base_rod +The rear/right lower horizontal connecting cylinder. +* **Component Purpose**: Connects the lower rear sections of the two side frames, completing the rigid triangular base structure. +* **Assembly Direction**: Inserted horizontally along the Y axis between the frames at support point (740, 0, 45). +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Inserted into the lower-rear circular cutouts of both side frames. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **upper_support_rod -> front_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the upper hole of the front frame. +* **upper_support_rod -> rear_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the upper hole of the rear frame. +* **left_base_rod -> front_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the lower-front hole of the front frame. +* **left_base_rod -> rear_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the lower-front hole of the rear frame. +* **right_base_rod -> front_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the lower-rear hole of the front frame. +* **right_base_rod -> rear_side_frame** | Joint: Dowel Joint | Note: Rod end inserted into the lower-rear hole of the rear frame. diff --git a/eval/tasks/muse-over_the_door_hook/harness.ts b/eval/tasks/muse-over_the_door_hook/harness.ts new file mode 100644 index 000000000..3f8534294 --- /dev/null +++ b/eval/tasks/muse-over_the_door_hook/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-over_the_door_hook/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'over_the_door_hook' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-over_the_door_hook'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-over_the_door_hook/prompt.md b/eval/tasks/muse-over_the_door_hook/prompt.md new file mode 100644 index 000000000..9d3964995 --- /dev/null +++ b/eval/tasks/muse-over_the_door_hook/prompt.md @@ -0,0 +1,110 @@ +# over_the_door_hook (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct an over-the-door hook rack featuring a main mounting rail and multiple evenly spaced hooks, designed for hanging garments, towels, or accessories without requiring permanent wall installation. + +## Geometry and Dimensions +Approx. 142.0 mm × 860.0 mm × 300.0 mm. + +## Material +Aluminum + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Load-bearing storage (supporting the weight of multiple hanging items such as heavy coats or bags). + +## Structural Features +Main door rail (U-channel profile); 10 hook tubes; 10 spherical end caps. + +## Special Requirements +Ensure the inner U-channel profile maintains strict dimensional accuracy to fit standard door thicknesses without excessive play. + +## Planned Component Quantity +21 + +## Component Names +- hook_tube_01 +- hook_tube_02 +- hook_tube_03 +- hook_tube_04 +- hook_tube_05 +- hook_tube_06 +- hook_tube_07 +- hook_tube_08 +- hook_tube_09 +- hook_tube_10 +- hook_end_cap_01 +- hook_end_cap_02 +- hook_end_cap_03 +- hook_end_cap_04 +- hook_end_cap_05 +- hook_end_cap_06 +- hook_end_cap_07 +- hook_end_cap_08 +- hook_end_cap_09 +- hook_end_cap_10 +- door_rail + +## Adjustable Parameters +- **rail_width**: 80 (40.0 ~ 140.0 mm). Defines the inner width of the U-channel to accommodate various standard door thicknesses. +- **rail_drop**: -300 (-500.0 ~ -120.0 mm). Determines the vertical reach of the rail down the face of the door for accessible hanging height. +- **rail_wall**: 3 (1.0 ~ 8.0 mm). Controls the structural thickness of the rail to ensure sufficient load-bearing stiffness and prevent bending. +- **front_rail_depth**: 60 (30.0 ~ 120.0 mm). Sets the length of the front section of the rail where the hooks are mounted. +- **rear_rail_offset**: 800 (300.0 ~ 1200.0 mm). Determines the overall span and spacing capacity of the rail across the width of the door. +- **hook_count**: 10 (4 ~ 18). The total number of hooks distributed evenly along the rail. +- **hook_pipe_radius**: 5 (2.0 ~ 12.0 mm). Defines the thickness of the hook tubes to prevent deformation under heavy loads. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~10. Hook Tubes (hook_tube_01 to hook_tube_10) +The primary hanging interfaces of the rack. +* **Component Purpose**: Extends outward from the rail and curves downwards to provide a secure resting point for hanging items. +* **Assembly Direction**: Horizontal extension along the +X axis, curving into the -Z axis. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions). The base of each tube is permanently affixed to the front face of the door rail. + +### 11~20. Hook End Caps (hook_end_cap_01 to hook_end_cap_10) +The safety terminations for the hooks. +* **Component Purpose**: Provides a smooth, spherical end to each hook tube to prevent snagging, tearing of fabrics, or user injury. +* **Assembly Direction**: Positioned at the distal end of each hook tube. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions). Mated to the trimmed ends of the hook tubes. + +### 21. Door Rail (door_rail) +The structural backbone of the assembly. +* **Component Purpose**: Acts as the main load-bearing base, hooking over the top edge of a door and providing a mounting surface for all hook tubes. +* **Assembly Direction**: Placed vertically over the door edge (spanning the Y-axis). +* **Connection & Kinematics**: Support Base. Acts as the core hub; all hooks are attached to its front face. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 21-component model: + +* **hook_tube_01 ~ 10 -> door_rail** | Joint: Bonding (Glue) | Note: Base of each hook tube is fixed to the front face of the door rail at evenly spaced intervals. +* **hook_end_cap_01 ~ 10 -> hook_tube_01 ~ 10** | Joint: Bonding (Glue) | Note: Each spherical cap is attached to the distal end of its corresponding hook tube. +* **door_rail -> All Components** | Joint: Support Base | Note: Acts as the central structural hub supporting the entire hook array. diff --git a/eval/tasks/muse-pegboard_circle/harness.ts b/eval/tasks/muse-pegboard_circle/harness.ts new file mode 100644 index 000000000..73407c9bc --- /dev/null +++ b/eval/tasks/muse-pegboard_circle/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_circle/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_circle' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_circle'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_circle/prompt.md b/eval/tasks/muse-pegboard_circle/prompt.md new file mode 100644 index 000000000..676c70960 --- /dev/null +++ b/eval/tasks/muse-pegboard_circle/prompt.md @@ -0,0 +1,72 @@ +# pegboard_circle (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a circular perforated pegboard panel designed for modular wall-mounted storage and organization. + +## Geometry and Dimensions +Approx. 400.0 mm × 400.0 mm × 5.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Load-bearing storage and wall-mounted organization for hanging tools or accessories. + +## Structural Features +Circular main panel; uniform grid of circular cutouts (holes). + +## Special Requirements +Keep assembly split unchanged. Ensure all grid holes maintain a minimum structural margin (`hole_radius + 0.5`) from the outer circular boundary to prevent edge breakout. + +## Planned Component Quantity +1 + +## Component Names +- perforated_panel + +## Adjustable Parameters +- **diameter**: 400 (150.0 ~ 800.0 mm). Defines the overall footprint and bounding box of the circular board. +- **thickness**: 5 (3.0 ~ 24.0 mm). Determines the structural rigidity of the board and the insertion depth for external pegs. +- **spacing**: 25 (10.0 ~ 50.0 mm). Controls the grid density and the center-to-center distance between adjacent holes. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sets the size of the cutouts to match standard external pegs or dowels. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main circular profile of the part based on the `diameter` parameter. +2. Generate a grid of cylindrical cutters and filter out any cylinders that intersect or exceed the outer boundary margin. +3. Execute a boolean cut to subtract the cylinder grid from the main board. + +--- + +### 1. perforated_panel +The main structural body of the pegboard. +* **Component Purpose**: Acts as the base platform, providing a standardized grid of sockets for attaching external hooks, pegs, or dowels. +* **Assembly Direction**: Modeled in the X-Y plane; typically mounted vertically against a wall surface. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). The internal holes act as female sockets for external male dowels/pegs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the single-component model: + +* **[External Pegs/Accessories] -> perforated_panel** | Joint: Dowel Joint | Note: External components are inserted into the grid of cylindrical holes on the panel. diff --git a/eval/tasks/muse-pegboard_frame/harness.ts b/eval/tasks/muse-pegboard_frame/harness.ts new file mode 100644 index 000000000..577f4ecf4 --- /dev/null +++ b/eval/tasks/muse-pegboard_frame/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_frame/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_frame' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_frame'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_frame/prompt.md b/eval/tasks/muse-pegboard_frame/prompt.md new file mode 100644 index 000000000..96157c262 --- /dev/null +++ b/eval/tasks/muse-pegboard_frame/prompt.md @@ -0,0 +1,76 @@ +# pegboard_frame (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a framed pegboard panel designed for wall-mounted storage and organization, featuring a recessed perforated grid for inserting hooks, pegs, or tool holders. + +## Geometry and Dimensions +Approx. 500.0 mm × 500.0 mm × 15.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage for hanging tools and accessories. + +## Structural Features +Thick outer support frame; recessed inner panel; uniform grid of cylindrical mounting holes. + +## Special Requirements +Keep assembly split unchanged. Ensure the inner pocket machining does not compromise the structural integrity of the outer frame. + +## Planned Component Quantity +1 + +## Component Names +- framed_panel + +## Adjustable Parameters +- **width**: 500 (200.0 ~ 800.0 mm). Determines the overall horizontal span of the pegboard. +- **height**: 500 (200.0 ~ 800.0 mm). Determines the overall vertical span of the pegboard. +- **thickness**: 5 (3.0 ~ 12.0 mm). Thickness of the inner perforated panel; must be thick enough to support hanging loads without excessive deflection. +- **frame_width**: 30 (15.0 ~ 60.0 mm). Width of the solid outer border, providing structural rigidity and a mounting surface. +- **frame_height**: 15 (8.0 ~ 30.0 mm). Total depth of the frame, creating a necessary standoff distance from the wall to allow peg insertion. +- **spacing**: 25 (10.0 ~ 50.0 mm). Center-to-center distance between the grid holes, dictating accessory compatibility. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Radius of the mounting holes, sized to accommodate standard pegboard hooks (acting as tenons). + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. framed_panel +The singular main body of the pegboard structure. +* **Component Purpose**: Acts as the primary load-bearing base, providing a rigid standoff frame for wall mounting and a grid of sockets (holes) for organizing external accessories. +* **Assembly Direction**: Fixed base component, typically mounted vertically against a wall. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The grid of cylindrical holes acts as an array of mortises designed to receive external pegs/hooks (tenons). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the single-component model interacting with external accessories: + +* **[External Pegs/Hooks] -> framed_panel** | Joint: interlocking | Note: External accessory tenons insert into the grid holes (mortises) of the inner panel. diff --git a/eval/tasks/muse-pegboard_hex_board/harness.ts b/eval/tasks/muse-pegboard_hex_board/harness.ts new file mode 100644 index 000000000..4b440c440 --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_board/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_hex_board/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_hex_board' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_hex_board'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_hex_board/prompt.md b/eval/tasks/muse-pegboard_hex_board/prompt.md new file mode 100644 index 000000000..d88ebcdf1 --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_board/prompt.md @@ -0,0 +1,72 @@ +# pegboard_hex_board (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a hexagonal perforated pegboard designed for modular storage, organization, and tool display. + +## Geometry and Dimensions +Approx. 400.0 mm × 346.4 mm × 5.0 mm. + +## Material +Timber + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Wall-mounted or stand-alone load-bearing storage and organization. + +## Structural Features +Hexagonal main panel; hexagonal grid array of circular holes. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +1 + +## Component Names +- Perforated panel + +## Adjustable Parameters +- **hex_size**: 200 (100.0 ~ 400.0 mm). Determines the overall footprint and storage capacity of the pegboard. +- **thickness**: 5 (3.0 ~ 24.0 mm). Controls the structural rigidity of the board and the insertion depth for external pegs. +- **spacing**: 25 (10.0 ~ 50.0 mm). Defines the density of the hole grid, affecting modular compatibility and structural integrity between holes. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sets the size of the sockets to match standard external pegs, hooks, or dowels. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main hexagonal profile of the part based on the original script. +2. Complete key features by cutting the hexagonal grid array of circular holes. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Perforated Panel +The central hub and sole component of the model. +* **Component Purpose**: Acts as the main structural base and provides a standardized grid of sockets for attaching external hooks, pegs, or modular fixtures. +* **Assembly Direction**: Base component, generated flat on the XY plane and extruded along the +Z axis. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features an array of circular holes acting as mortises (sockets) for external components. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **External Attachments -> Perforated Panel** | Joint: interlocking | Note: External pegs or hooks (not included in the model) are inserted into the board's circular grid holes. diff --git a/eval/tasks/muse-pegboard_hex_stand/harness.ts b/eval/tasks/muse-pegboard_hex_stand/harness.ts new file mode 100644 index 000000000..047095489 --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_stand/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_hex_stand/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_hex_stand' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_hex_stand'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_hex_stand/prompt.md b/eval/tasks/muse-pegboard_hex_stand/prompt.md new file mode 100644 index 000000000..d9dadb117 --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_stand/prompt.md @@ -0,0 +1,82 @@ +# pegboard_hex_stand (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a desktop hexagonal pegboard stand designed for organizing and displaying small items via a perforated grid. + +## Geometry and Dimensions +Approx. 400.0 mm × 120.0 mm × 366.4 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Desktop storage and display, light load-bearing. + +## Structural Features +Hexagonal perforated panel; rectangular support base. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +2 + +## Component Names +- perforated_panel +- base + +## Adjustable Parameters +- **hex_size**: 200 (100.0 ~ 350.0 mm). Determines the overall display area and bounding width of the hexagonal pegboard. +- **panel_thickness**: 8 (5.0 ~ 15.0 mm). Controls the structural rigidity of the pegboard and the thickness of the connecting tenon. +- **spacing**: 25 (10.0 ~ 50.0 mm). Defines the distance between adjacent peg holes in the hexagonal grid pattern. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sets the size of the holes to accommodate standard pegboard hooks or dowels. +- **base_depth**: 120 (60.0 ~ 200.0 mm). Ensures the stand's anti-overturning stability on a flat surface. +- **base_height**: 20 (10.0 ~ 40.0 mm). Provides sufficient depth for the mortise slot to securely hold the panel's tenon. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. perforated_panel +The main vertical display surface of the stand. +* **Component Purpose**: Provides a grid of holes for attaching hooks or accessories, acting as the primary functional area. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom edge features a protruding rectangular tenon. + +### 2. base +The supporting foundation of the stand. +* **Component Purpose**: Acts as the stable base, preventing the vertical panel from tipping over under eccentric loads. +* **Assembly Direction**: Fixed base component, positioned at the bottom (absolute Z from 0 to `base_height`). +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The top surface features a central rectangular slot (mortise) matching the panel's tenon. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **perforated_panel -> base** | Joint: interlocking | Note: Panel bottom tenon inserted into the base's top slot. diff --git a/eval/tasks/muse-pegboard_hex_table/harness.ts b/eval/tasks/muse-pegboard_hex_table/harness.ts new file mode 100644 index 000000000..1c33e23dc --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_table/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_hex_table/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_hex_table' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_hex_table'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_hex_table/prompt.md b/eval/tasks/muse-pegboard_hex_table/prompt.md new file mode 100644 index 000000000..e779493d4 --- /dev/null +++ b/eval/tasks/muse-pegboard_hex_table/prompt.md @@ -0,0 +1,87 @@ +# pegboard_hex_table (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a three-legged hexagonal pegboard table designed for modular storage, display, and organization. + +## Geometry and Dimensions +Approx. 400.0 mm × 346.4 mm × 308.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage and lightweight display stand. + +## Structural Features +Hexagonal perforated tabletop panel; three vertical support legs. + +## Special Requirements +Keep assembly split unchanged. Ensure pegboard holes do not intersect or interfere with the leg mortise sockets. + +## Planned Component Quantity +4 + +## Component Names +- perforated_panel +- leg_01 +- leg_02 +- leg_03 + +## Adjustable Parameters +- **hex_size**: 200 (120.0 ~ 350.0 mm). Determines the overall tabletop area and footprint radius. +- **thickness**: 8 (5.0 ~ 15.0 mm). Controls the structural rigidity and load-bearing capacity of the tabletop panel. +- **spacing**: 25 (10.0 ~ 50.0 mm). Defines the density of the pegboard hole grid for accessory placement. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sets the size of the pegboard holes to ensure compatibility with standard insertion pegs. +- **leg_height**: 300 (150.0 ~ 500.0 mm). Determines the elevation of the table surface from the ground. +- **leg_thickness**: 30 (20.0 ~ 50.0 mm). Ensures adequate vertical load-bearing stiffness and stability for the legs. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. perforated_panel +The central functional surface of the table. +* **Component Purpose**: Acts as the main load-bearing base for storage, provides a grid of holes for pegboard accessories, and houses the mechanical interfaces (sockets) for the legs. +* **Assembly Direction**: Fixed base component, positioned horizontally at absolute Z = `leg_height`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The underside features three rectangular sockets distributed at 120-degree intervals to receive the legs. + +### 2~4. Three Legs (leg_01, leg_02, leg_03) +The supporting entities of the table. +* **Component Purpose**: Vertical support. Transfers the tabletop load to the ground, ensuring anti-overturning stability in the X-Y plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the perforated panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The top of each leg features a rectangular tenon that interference-fits into the corresponding bottom sockets of the perforated panel. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 4-component model: + +* **leg_01 -> perforated_panel** | Joint: interlocking | Note: Leg top tenon inserted into the panel's first socket (at 30 degrees). +* **leg_02 -> perforated_panel** | Joint: interlocking | Note: Leg top tenon inserted into the panel's second socket (at 150 degrees). +* **leg_03 -> perforated_panel** | Joint: interlocking | Note: Leg top tenon inserted into the panel's third socket (at 270 degrees). +* **perforated_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-pegboard_slot/harness.ts b/eval/tasks/muse-pegboard_slot/harness.ts new file mode 100644 index 000000000..bc5dce15a --- /dev/null +++ b/eval/tasks/muse-pegboard_slot/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_slot/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_slot' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_slot'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_slot/prompt.md b/eval/tasks/muse-pegboard_slot/prompt.md new file mode 100644 index 000000000..e3894eb04 --- /dev/null +++ b/eval/tasks/muse-pegboard_slot/prompt.md @@ -0,0 +1,76 @@ +# pegboard_slot (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a flat perforated pegboard panel featuring a regular grid of slots, designed for modular wall storage and tool organization. + +## Geometry and Dimensions +Approx. 600.0 mm × 400.0 mm × 5.0 mm. + +## Material +Timber + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Load-bearing storage (hanging tools, shelves, and accessories via external hooks). + +## Structural Features +Flat rectangular panel; rounded outer corners; regular grid of vertical slots. + +## Special Requirements +Ensure the slot grid remains centered relative to the outer board dimensions regardless of parameter adjustments. + +## Planned Component Quantity +1 + +## Component Names +- perforated_panel + +## Adjustable Parameters +- **width**: 600 (200.0 ~ 1200.0 mm). Defines the overall horizontal coverage area of the pegboard. +- **height**: 400 (200.0 ~ 1200.0 mm). Defines the overall vertical coverage area of the pegboard. +- **thickness**: 5 (3.0 ~ 24.0 mm). Determines the board's structural rigidity and compatibility with the insertion depth of external hooks. +- **board_corner_radius**: 10 (0.0 ~ 40.0 mm). Eliminates sharp corners to prevent injury during handling and installation. +- **spacing**: 25 (10.0 ~ 50.0 mm). Standardizes the modular grid pitch to ensure compatibility with standardized pegboard accessories. +- **slot_length**: 15 (6.0 ~ 30.0 mm). Defines the vertical opening size for inserting flat or angled hooks. +- **slot_width**: 6 (3.0 ~ 12.0 mm). Defines the horizontal opening size, ensuring a snug fit for the inserted tenons/hooks. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main rectangular profile of the board with rounded corners. +2. Generate the 2D grid of slot cutters based on the calculated rows, columns, and spacing. +3. Execute a boolean cut to subtract the slot grid from the main board body. + +--- + +### 1. perforated_panel +The main structural body of the pegboard. +* **Component Purpose**: Acts as a modular mounting base, providing a standardized grid of sockets (slots) to support external hanging accessories. +* **Assembly Direction**: Fixed base component, typically mounted vertically against a wall. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The internal slots act as mortises to receive the tenons of external hooks and brackets. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the single-component model and its intended use: + +* **External Accessories -> perforated_panel** | Joint: interlocking | Note: External hooks/tenons are inserted into the panel's grid of slots to form a rigid or semi-rigid hanging connection. diff --git a/eval/tasks/muse-pegboard_stand/harness.ts b/eval/tasks/muse-pegboard_stand/harness.ts new file mode 100644 index 000000000..bcaf3691b --- /dev/null +++ b/eval/tasks/muse-pegboard_stand/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_stand/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_stand' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_stand'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_stand/prompt.md b/eval/tasks/muse-pegboard_stand/prompt.md new file mode 100644 index 000000000..256ac33a4 --- /dev/null +++ b/eval/tasks/muse-pegboard_stand/prompt.md @@ -0,0 +1,84 @@ +# pegboard_stand (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a freestanding pegboard stand designed for desktop organization, tool storage, and display. + +## Geometry and Dimensions +Approx. 400.0 mm × 120.0 mm × 520.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Desktop storage and display; load-bearing for hanging small tools, accessories, or stationery. + +## Structural Features +Perforated vertical panel; horizontal stabilizing base. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +2 + +## Component Names +- perforated_panel +- base + +## Adjustable Parameters +- **width**: 400 (200.0 ~ 800.0 mm). Determines the overall width of the pegboard and base. +- **height**: 500 (300.0 ~ 800.0 mm). Determines the vertical storage area of the pegboard. +- **panel_thickness**: 8 (5.0 ~ 15.0 mm). Ensures sufficient structural stiffness for hanging items without excessive weight or material use. +- **spacing**: 25 (10.0 ~ 50.0 mm). Controls the density of the pegboard hole grid for accessory placement. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sized to fit standard pegboard hooks and pegs. +- **base_depth**: 120 (60.0 ~ 200.0 mm). Provides anti-overturning stability in the front-to-back (Y-axis) direction. +- **base_height**: 20 (10.0 ~ 40.0 mm). Provides enough thickness to accommodate the blind mortise slot and ensure base rigidity. +- **board_corner_radius**: 8 (0.0 ~ 30.0 mm). Intended to round the sharp edges of the board for safety and aesthetics. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. perforated_panel +The main vertical storage interface of the stand. +* **Component Purpose**: Provides a standardized grid of through-holes for attaching hooks, shelves, and mounts to hold items. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom edge features a centered rectangular tenon that fits into the base. + +### 2. base +The horizontal support structure. +* **Component Purpose**: Acts as the stabilizing footprint to transfer the load to the desktop and prevent the vertical panel from tipping over under eccentric loads. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = 0$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features a central blind slot (mortise) on its top face to receive the panel's tenon. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 2-component model: + +* **perforated_panel -> base** | Joint: interlocking | Note: Panel bottom tenon inserted into the base's top central slot. diff --git a/eval/tasks/muse-pegboard_tray/harness.ts b/eval/tasks/muse-pegboard_tray/harness.ts new file mode 100644 index 000000000..e404527a0 --- /dev/null +++ b/eval/tasks/muse-pegboard_tray/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pegboard_tray/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pegboard_tray' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pegboard_tray'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pegboard_tray/prompt.md b/eval/tasks/muse-pegboard_tray/prompt.md new file mode 100644 index 000000000..e6a9e4f92 --- /dev/null +++ b/eval/tasks/muse-pegboard_tray/prompt.md @@ -0,0 +1,63 @@ +# pegboard_tray (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a rectangular storage tray featuring a raised perimeter lip and a standardized pegboard hole grid on the base for organizing tools and components. + +## Geometry and Dimensions +Approx. 600.0 mm × 400.0 mm × 25.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Not applicable (Single monolithic component) + +## Mechanical Condition +Load-bearing storage and organization of tools or hardware items on a flat surface. + +## Structural Features +Perforated base panel; four raised perimeter lips (front, back, left, right). + +## Special Requirements +The component must be machined or formed as a single continuous solid body. + +## Planned Component Quantity +1 + +## Component Names +- tray_panel + +## Adjustable Parameters +- **width**: 600 (200.0 ~ 1200.0 mm). Defines the overall X-axis dimension of the tray. +- **height**: 400 (200.0 ~ 1200.0 mm). Defines the overall Y-axis dimension (depth) of the tray. +- **thickness**: 5 (3.0 ~ 24.0 mm). Determines the structural thickness of the bottom pegboard panel to support the load. +- **spacing**: 25 (10.0 ~ 50.0 mm). Controls the center-to-center distance between the pegboard holes, ensuring compatibility with standard peg hooks. +- **hole_radius**: 3 (1.0 ~ 8.0 mm). Sets the size of the pegboard holes; must match the intended insertion hardware. +- **lip_thickness**: 5 (3.0 ~ 15.0 mm). Defines the wall thickness of the perimeter lips to provide adequate edge rigidity. +- **lip_height**: 20 (10.0 ~ 50.0 mm). Sets the height of the perimeter lips above the base panel to effectively contain loose items. + +## Component Details + +### 1. tray_panel +The main and only body of the pegboard tray. +* **Component Purpose**: Provides a flat, perforated surface for peg insertion and a raised boundary to prevent items from falling off the edges. +* **Assembly Direction**: Not applicable (Standalone part). +* **Connection & Kinematics**: Not applicable (Single component). + +--- + +## Component Assembly Graph (Textual) +tray_panel -> Standalone | Joint: None | Note: Single monolithic component. diff --git a/eval/tasks/muse-pen_holder_glue_double_caddy/harness.ts b/eval/tasks/muse-pen_holder_glue_double_caddy/harness.ts new file mode 100644 index 000000000..24db5e652 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_double_caddy/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_glue_double_caddy/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_glue_double_caddy' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_glue_double_caddy'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_glue_double_caddy/prompt.md b/eval/tasks/muse-pen_holder_glue_double_caddy/prompt.md new file mode 100644 index 000000000..364a990f5 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_double_caddy/prompt.md @@ -0,0 +1,116 @@ +# pen_holder_glue_double_caddy (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a glued rectangular desk caddy with two compartments for organizing pens and stationery. + +## Geometry and Dimensions +Approx. 126.0 mm × 78.0 mm × 112.0 mm. + +## Material +Acrylic + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Desktop storage, stationary load-bearing for pens and small tools. + +## Structural Features +Base panel; front and back panels; left and right side panels; center divider. + +## Special Requirements +Keep assembly split unchanged. Ensure all mating surfaces are clean and flat for optimal solvent welding/glue adhesion. + +## Planned Component Quantity +6 + +## Component Names +- base_panel +- front_panel +- back_panel +- left_panel +- right_panel +- center_divider + +## Adjustable Parameters +- **outer_width**: 126.0 (110.0 ~ 150.0 mm). Controls the overall width of the caddy. +- **outer_depth**: 78.0 (62.0 ~ 102.0 mm). Controls the overall depth of the caddy. +- **wall_height**: 108.0 (80.0 ~ 148.0 mm). Determines the internal storage depth for the compartments. +- **wall_thickness**: 3.0 (2.0 ~ 5.0 mm). Defines the thickness of the vertical panels, constrained by standard laser cutting sheet thicknesses. +- **base_thickness**: 4.0 (2.8 ~ 6.0 mm). Defines the thickness of the bottom panel to ensure structural stability and a solid gluing foundation. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. base_panel +The foundational platform of the caddy. +* **Component Purpose**: Acts as the bottom support, holding the stored items and providing a flat base for gluing the vertical walls. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = 0. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 2. front_panel +The front boundary of the caddy. +* **Component Purpose**: Retains items within the front of the compartments and provides structural rigidity to the side panels. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the front edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the inner faces of the left and right panels. + +### 3. back_panel +The rear boundary of the caddy. +* **Component Purpose**: Retains items within the rear of the compartments and provides structural rigidity. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the rear edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the inner faces of the left and right panels. + +### 4. left_panel +The left boundary of the caddy. +* **Component Purpose**: Encloses the left side, spanning the full depth of the caddy to cap the front and back panels. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the left edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the end faces of the front and back panels. + +### 5. right_panel +The right boundary of the caddy. +* **Component Purpose**: Encloses the right side, spanning the full depth of the caddy to cap the front and back panels. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the right edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the end faces of the front and back panels. + +### 6. center_divider +The internal partition of the caddy. +* **Component Purpose**: Divides the internal volume into two separate compartments for organization. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the center of the base panel. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the inner faces of the front and back panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 6-component model: + +* **front_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the front top surface of the base panel. +* **back_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the rear top surface of the base panel. +* **left_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the left top surface of the base panel. +* **right_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the right top surface of the base panel. +* **center_divider -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the center top surface of the base panel. +* **front_panel -> left_panel & right_panel** | Joint: Bonding (Glue) | Note: Side edges of the front panel are glued to the inner faces of the left and right panels. +* **back_panel -> left_panel & right_panel** | Joint: Bonding (Glue) | Note: Side edges of the back panel are glued to the inner faces of the left and right panels. +* **center_divider -> front_panel & back_panel** | Joint: Bonding (Glue) | Note: Ends of the divider are glued to the inner faces of the front and back panels. diff --git a/eval/tasks/muse-pen_holder_glue_square_scoop/harness.ts b/eval/tasks/muse-pen_holder_glue_square_scoop/harness.ts new file mode 100644 index 000000000..6cc2c7b24 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_square_scoop/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_glue_square_scoop/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_glue_square_scoop' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_glue_square_scoop'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_glue_square_scoop/prompt.md b/eval/tasks/muse-pen_holder_glue_square_scoop/prompt.md new file mode 100644 index 000000000..7255bda68 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_square_scoop/prompt.md @@ -0,0 +1,109 @@ +# pen_holder_glue_square_scoop (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a square pen cup with a lowered front scoop, designed to be assembled from flat panels using adhesive. + +## Geometry and Dimensions +Approx. 84.0 mm × 84.0 mm × 116.0 mm. + +## Material +Acrylic + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Desktop storage for holding pens, pencils, and other stationery items. + +## Structural Features +Base panel; front panel with notch; back panel; left panel; right panel. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +5 + +## Component Names +- base_panel +- front_panel +- back_panel +- left_panel +- right_panel + +## Adjustable Parameters +- **outer_width**: 84.0 (68.0 ~ 108.0 mm). Determines the overall width of the pen holder. +- **outer_depth**: 84.0 (68.0 ~ 108.0 mm). Determines the overall depth of the pen holder. +- **wall_height**: 112.0 (84.0 ~ 152.0 mm). Controls the internal storage height for the pens. +- **wall_thickness**: 3.0 (2.0 ~ 5.0 mm). Matches standard sheet material thickness suitable for laser cutting. +- **base_thickness**: 4.0 (2.8 ~ 6.0 mm). Provides a stable and sufficiently heavy bottom foundation for the holder. +- **front_notch_width**: 34.0 (18.0 ~ 58.0 mm). Sets the width of the front access scoop. +- **front_notch_height**: 28.0 (50.0 ~ 68.0 mm). Sets the height of the front access scoop to allow easy retrieval of shorter items. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. base_panel +The bottom foundation of the pen holder. +* **Component Purpose**: Acts as the floor of the container and provides a flat base for attaching the vertical walls. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = 0. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 2. front_panel +The front-facing wall with a scoop. +* **Component Purpose**: Retains items while providing a lowered rectangular notch for easy access to shorter stationery. +* **Assembly Direction**: Placed vertically on the front edge of the base panel along the +Z axis. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 3. back_panel +The rear wall of the pen holder. +* **Component Purpose**: Encloses the back of the storage volume to keep tall items upright. +* **Assembly Direction**: Placed vertically on the rear edge of the base panel along the +Z axis. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 4. left_panel +The left side wall. +* **Component Purpose**: Encloses the left side of the storage volume, spanning the full outer depth. +* **Assembly Direction**: Placed vertically on the left edge of the base panel along the +Z axis. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 5. right_panel +The right side wall. +* **Component Purpose**: Encloses the right side of the storage volume, spanning the full outer depth. +* **Assembly Direction**: Placed vertically on the right edge of the base panel along the +Z axis. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **left_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the left top surface of the base. +* **right_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the right top surface of the base. +* **front_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the front top surface of the base. +* **back_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the rear top surface of the base. +* **front_panel -> left_panel & right_panel** | Joint: Bonding (Glue) | Note: Side edges of the front panel are glued to the inner faces of the left and right panels. +* **back_panel -> left_panel & right_panel** | Joint: Bonding (Glue) | Note: Side edges of the back panel are glued to the inner faces of the left and right panels. diff --git a/eval/tasks/muse-pen_holder_glue_window_box/harness.ts b/eval/tasks/muse-pen_holder_glue_window_box/harness.ts new file mode 100644 index 000000000..f83c4df65 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_window_box/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_glue_window_box/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_glue_window_box' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_glue_window_box'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_glue_window_box/prompt.md b/eval/tasks/muse-pen_holder_glue_window_box/prompt.md new file mode 100644 index 000000000..2ef260005 --- /dev/null +++ b/eval/tasks/muse-pen_holder_glue_window_box/prompt.md @@ -0,0 +1,110 @@ +# pen_holder_glue_window_box (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a glued desk pen box with lightened side windows for storing pens and desktop stationery. + +## Geometry and Dimensions +Approx. 92.0 mm × 76.0 mm × 118.0 mm. + +## Material +Timber + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +Bonding (Glue) + +## Mechanical Condition +Desktop storage, stationary, light load-bearing. + +## Structural Features +Base panel; front panel; back panel; left panel; right panel. + +## Special Requirements +Keep assembly split unchanged. Ensure mating surfaces are clean and flat for optimal glue adhesion. + +## Planned Component Quantity +5 + +## Component Names +- base_panel +- front_panel +- back_panel +- left_panel +- right_panel + +## Adjustable Parameters +- **outer_width**: 92.0 (76.0 ~ 116.0 mm). Controls the overall width of the pen box. +- **outer_depth**: 76.0 (60.0 ~ 100.0 mm). Controls the overall depth of the pen box. +- **wall_height**: 114.0 (86.0 ~ 154.0 mm). Determines the internal storage height for pens and stationery. +- **wall_thickness**: 3.0 (2.0 ~ 5.0 mm). Defines the structural thickness of the vertical side panels. +- **base_thickness**: 4.0 (2.8 ~ 6.0 mm). Defines the thickness of the bottom load-bearing panel. +- **window_width**: 42.0 (26.0 ~ 66.0 mm). Width of the side and back cutouts for weight reduction and visibility. +- **window_height**: 50.0 (50.0 ~ 90.0 mm). Height of the side and back cutouts. +- **window_bottom**: 28.0 (18.0 ~ 42.0 mm). Controls the vertical offset of the windows from the base to retain items at the bottom. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. base_panel +The foundational support of the pen box. +* **Component Purpose**: Acts as the main load-bearing bottom and provides a flat surface for attaching the vertical walls. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = 0. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). + +### 2. front_panel +The front enclosure of the box. +* **Component Purpose**: Retains the stored items from the front. Spans the inner width between the left and right panels. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the front edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the inner faces of the side panels. + +### 3. back_panel +The rear enclosure of the box. +* **Component Purpose**: Retains the stored items from the back. Features a central window cutout for visibility and material reduction. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the rear edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and the inner faces of the side panels. + +### 4. left_panel +The left side enclosure of the box. +* **Component Purpose**: Provides lateral support and encloses the left side. Features a central window cutout. Spans the full outer depth. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the left edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and caps the edges of the front and back panels. + +### 5. right_panel +The right side enclosure of the box. +* **Component Purpose**: Provides lateral support and encloses the right side. Features a central window cutout. Spans the full outer depth. +* **Assembly Direction**: Placed vertically along the +Z axis, resting on the base panel at the right edge. +* **Connection & Kinematics**: Bonding (Glue) (Fully constrained in all directions (permanent)). Glued to the base panel and caps the edges of the front and back panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 5-component model: + +* **front_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the front top surface of the base panel. +* **back_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the rear top surface of the base panel. +* **left_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the left top surface of the base panel. +* **right_panel -> base_panel** | Joint: Bonding (Glue) | Note: Bottom edge glued to the right top surface of the base panel. +* **left_panel -> front_panel & back_panel** | Joint: Bonding (Glue) | Note: Inner face of the left panel glued to the left edges of the front and back panels. +* **right_panel -> front_panel & back_panel** | Joint: Bonding (Glue) | Note: Inner face of the right panel glued to the right edges of the front and back panels. diff --git a/eval/tasks/muse-pen_holder_print_fluted_square/harness.ts b/eval/tasks/muse-pen_holder_print_fluted_square/harness.ts new file mode 100644 index 000000000..17186b97b --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_fluted_square/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_print_fluted_square/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_print_fluted_square' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_print_fluted_square'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_print_fluted_square/prompt.md b/eval/tasks/muse-pen_holder_print_fluted_square/prompt.md new file mode 100644 index 000000000..42a276b89 --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_fluted_square/prompt.md @@ -0,0 +1,78 @@ +# pen_holder_print_fluted_square (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a one-piece 3D printed fluted square pen cup for desktop stationery storage. + +## Geometry and Dimensions +Approx. 86.0 mm × 86.0 mm × 116.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Desktop storage, non-load-bearing, holding lightweight stationery items. + +## Structural Features +Solid base; fluted outer walls; rounded corners; hollow interior cavity. + +## Special Requirements +Must be manufactured as a single continuous solid body; designed to be printed without internal supports. + +## Planned Component Quantity +1 + +## Component Names +- Printed body + +## Adjustable Parameters +- **outer_width**: 86.0 (70.0 ~ 110.0 mm). Defines the overall width of the pen holder footprint. +- **outer_depth**: 86.0 (70.0 ~ 110.0 mm). Defines the overall depth of the pen holder footprint. +- **wall_height**: 116.0 (88.0 ~ 156.0 mm). Determines the total height, ensuring pens are adequately supported without tipping over. +- **wall_thickness**: 3.2 (2.0 ~ 5.2 mm). Ensures structural rigidity of the walls during printing and daily use. +- **base_thickness**: 4.4 (3.2 ~ 6.4 mm). Provides a solid bottom to lower the center of gravity and support the resting stationery. +- **corner_radius**: 10.0 (18.0 ~ 34.0 mm). Controls the rounding of the vertical corners for aesthetics and ergonomic handling. +- **base_band_height**: 20.0 (50.0 ~ 60.0 mm). Defines the height of the un-fluted solid band at the bottom of the cup. +- **groove_width**: 2.1 (18.0 ~ 26.1 mm). Sets the width of the decorative vertical flutes on the exterior walls. +- **groove_depth**: 1.2 (18.0 ~ 25.2 mm). Sets the indentation depth of the decorative vertical flutes. +- **groove_pitch**: 5.0 (1.0 ~ 19.0 mm). Controls the spacing/frequency of the vertical flutes along the perimeter. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main rounded box profile of the part based on the original script. +2. Complete key features like the inner cavity subtraction and the exterior fluted groove cuts. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Printed body +The main and only component of the pen holder. +* **Component Purpose**: Acts as the storage container for pens, featuring a solid base and decorative fluted walls. +* **Assembly Direction**: N/A (Printed in place, typically built upwards along the +Z axis from the base). +* **Connection & Kinematics**: None (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **Printed body** | Joint: None | Note: Standalone single-piece component. diff --git a/eval/tasks/muse-pen_holder_print_round_ribbed/harness.ts b/eval/tasks/muse-pen_holder_print_round_ribbed/harness.ts new file mode 100644 index 000000000..ce5eb4369 --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_round_ribbed/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_print_round_ribbed/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_print_round_ribbed' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_print_round_ribbed'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_print_round_ribbed/prompt.md b/eval/tasks/muse-pen_holder_print_round_ribbed/prompt.md new file mode 100644 index 000000000..9fc8e1b08 --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_round_ribbed/prompt.md @@ -0,0 +1,74 @@ +# pen_holder_print_round_ribbed (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a one-piece 3D printed round pen cup with a ribbed outer wall for desktop stationery organization. + +## Geometry and Dimensions +Approx. 86.0 mm × 86.0 mm × 118.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Desktop storage, holding lightweight vertical loads (pens, pencils, markers) with a low center of gravity to prevent tipping. + +## Structural Features +Ribbed exterior wall (24-sided polygon); faceted interior cavity (56-sided polygon); solid integrated base. + +## Special Requirements +Must be manufactured as a single continuous solid body. Designed to be printed upright without the need for support structures. + +## Planned Component Quantity +1 + +## Component Names +- printed_body + +## Adjustable Parameters +- **outer_radius**: 43.0 (27.0 ~ 67.0 mm). Determines the overall footprint and internal storage capacity of the pen holder. +- **wall_height**: 118.0 (90.0 ~ 158.0 mm). Controls the depth of the cup to adequately support standard writing instruments without them falling out. +- **wall_thickness**: 3.2 (2.0 ~ 5.2 mm). Ensures structural rigidity of the vertical walls while optimizing print time and material consumption. +- **base_thickness**: 4.2 (3.0 ~ 6.2 mm). Provides a solid, weighted bottom to lower the center of gravity and prevent tipping. +- **outer_sides**: 24. Defines the ribbed aesthetic and tactile grip of the exterior wall. +- **inner_sides**: 56. Defines the relatively smooth interior surface to prevent pens from catching on sharp internal corners. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main outer profile of the part using a 24-sided polygon prism. +2. Cut the interior cavity using a 56-sided polygon prism offset by the wall thickness and base thickness. +3. Export the resulting monolithic shell. + +--- + +### 1. printed_body +The main and only structural entity of the pen holder. +* **Component Purpose**: Acts as the primary containment vessel, providing a stable base and vertical walls to organize desktop items. +* **Assembly Direction**: Not applicable (Manufactured in place from the base up along the +Z axis). +* **Connection & Kinematics**: Not applicable (Monolithic structure). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **printed_body -> Standalone** | Joint: None | Note: Monolithic 3D printed structure; no assembly required. diff --git a/eval/tasks/muse-pen_holder_print_tri_cluster/harness.ts b/eval/tasks/muse-pen_holder_print_tri_cluster/harness.ts new file mode 100644 index 000000000..ec02f65a8 --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_tri_cluster/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_print_tri_cluster/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_print_tri_cluster' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_print_tri_cluster'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_print_tri_cluster/prompt.md b/eval/tasks/muse-pen_holder_print_tri_cluster/prompt.md new file mode 100644 index 000000000..b110cb661 --- /dev/null +++ b/eval/tasks/muse-pen_holder_print_tri_cluster/prompt.md @@ -0,0 +1,60 @@ +# pen_holder_print_tri_cluster (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +A one-piece, three-cell clustered pen organizer designed for desktop stationery storage. + +## Geometry and Dimensions +Approx. 88.0 mm × 83.4 mm × 108.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Desktop storage, stationary load-bearing for pens, pencils, and other cylindrical stationery items. + +## Structural Features +Three clustered cylindrical cells; unified solid base; shared intersecting inner walls. + +## Special Requirements +Keep assembly split unchanged. The model must be exported as a single, continuous solid body to ensure printability without assembly. + +## Planned Component Quantity +1 + +## Component Names +- printed_body + +## Adjustable Parameters +- **cell_radius**: 24.0 (18.0 ~ 48.0 mm). Determines the internal storage capacity and diameter of each individual compartment. +- **wall_height**: 108.0 (80.0 ~ 148.0 mm). Controls the overall height of the organizer to adequately support various pen lengths without tipping over. +- **wall_thickness**: 3.0 (2.0 ~ 5.0 mm). Ensures structural rigidity and provides sufficient thickness for FDM wall line generation. +- **base_thickness**: 4.0 (2.8 ~ 6.0 mm). Provides a solid bottom to prevent items from falling through and adds lower-center-of-gravity weight for stability. +- **center_spacing**: 34.0 (18.0 ~ 58.0 mm). Defines the distance between the centers of the three cells, affecting the overall footprint and the degree of intersection/overlap between the cylinders. + +## Component Details + +### 1. printed_body +The main and only component of the pen organizer. +* **Component Purpose**: Acts as the complete structural and functional body, providing three distinct compartments for organizing pens. +* **Assembly Direction**: Not applicable (printed in place vertically from the base along the +Z axis). +* **Connection & Kinematics**: Not Applicable (Monolithic part; fully constrained internally). + +--- + +## Component Assembly Graph (Textual) +printed_body -> Standalone | Joint: None | Note: Monolithic single-piece 3D printed structure; no physical assembly required. diff --git a/eval/tasks/muse-pen_holder_tab_frame_square/harness.ts b/eval/tasks/muse-pen_holder_tab_frame_square/harness.ts new file mode 100644 index 000000000..d6baf8bb7 --- /dev/null +++ b/eval/tasks/muse-pen_holder_tab_frame_square/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_tab_frame_square/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_tab_frame_square' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_tab_frame_square'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_tab_frame_square/prompt.md b/eval/tasks/muse-pen_holder_tab_frame_square/prompt.md new file mode 100644 index 000000000..6acc4d9d9 --- /dev/null +++ b/eval/tasks/muse-pen_holder_tab_frame_square/prompt.md @@ -0,0 +1,119 @@ +# pen_holder_tab_frame_square (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a slot-assembled square pen holder with a top locking frame designed for desktop organization and stationery storage. + +## Geometry and Dimensions +Approx. 86.0 mm × 86.0 mm × 122.0 mm. + +## Material +Timber + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Desktop storage, holding pens, pencils, and other lightweight stationery items. + +## Structural Features +Base panel; four side wall panels (front, back, left, right); top locking frame. + +## Special Requirements +Keep assembly split unchanged. Ensure tight tolerances for interference fit during assembly. + +## Planned Component Quantity +6 + +## Component Names +- Base panel +- Front panel +- Back panel +- Left panel +- Right panel +- Top frame + +## Adjustable Parameters +- **outer_width**: 86.0 (70.0 ~ 110.0 mm). Controls the overall external width of the pen holder. +- **outer_depth**: 86.0 (70.0 ~ 110.0 mm). Controls the overall external depth of the pen holder. +- **wall_height**: 110.0 (82.0 ~ 150.0 mm). Determines the internal vertical storage space for pens. +- **wall_thickness**: 5.0 (3.8 ~ 7.0 mm). Defines the structural thickness of the four side panels. +- **base_thickness**: 6.0 (4.8 ~ 8.0 mm). Defines the thickness of the bottom load-bearing panel. +- **frame_thickness**: 6.0 (4.8 ~ 8.0 mm). Defines the thickness of the top locking frame. +- **tab_width**: 18.0 (18.0 ~ 42.0 mm). Determines the width of the interlocking tabs (tenons) used for assembly. +- **frame_border**: 11.0 (1.0 ~ 25.0 mm). Controls the width of the top frame's border, defining the size of the top opening. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Base panel +The bottom support of the pen holder. +* **Component Purpose**: Acts as the main load-bearing base, providing mortise slots (sockets) for the side panels to insert into. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = 0. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features rectangular slots along its perimeter to receive the bottom tabs of the side panels. + +### 2. Front panel +The front enclosure wall. +* **Component Purpose**: Encloses the front side of the storage volume and provides vertical structure. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs at the bottom for base insertion and at the top for the locking frame. + +### 3. Back panel +The rear enclosure wall. +* **Component Purpose**: Encloses the rear side of the storage volume and provides vertical structure. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs at the bottom for base insertion and at the top for the locking frame. + +### 4. Left panel +The left enclosure wall. +* **Component Purpose**: Encloses the left side of the storage volume and provides vertical structure. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs at the bottom for base insertion and at the top for the locking frame. + +### 5. Right panel +The right enclosure wall. +* **Component Purpose**: Encloses the right side of the storage volume and provides vertical structure. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features protruding tabs at the bottom for base insertion and at the top for the locking frame. + +### 6. Top frame +The upper locking collar. +* **Component Purpose**: Locks the four side panels together at the top, ensuring structural integrity and preventing the walls from splaying outward under load. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the assembled side panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features mortise slots along its inner border that receive the top tabs of all four side panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 6-component model: + +* **Front panel -> Base panel** | Joint: interlocking | Note: Front panel bottom tabs inserted into base panel's front slots. +* **Back panel -> Base panel** | Joint: interlocking | Note: Back panel bottom tabs inserted into base panel's rear slots. +* **Left panel -> Base panel** | Joint: interlocking | Note: Left panel bottom tabs inserted into base panel's left slots. +* **Right panel -> Base panel** | Joint: interlocking | Note: Right panel bottom tabs inserted into base panel's right slots. +* **Top frame -> Front panel** | Joint: interlocking | Note: Top frame's front slots receive front panel's top tabs. +* **Top frame -> Back panel** | Joint: interlocking | Note: Top frame's rear slots receive back panel's top tabs. +* **Top frame -> Left panel** | Joint: interlocking | Note: Top frame's left slots receive left panel's top tabs. +* **Top frame -> Right panel** | Joint: interlocking | Note: Top frame's right slots receive right panel's top tabs. diff --git a/eval/tasks/muse-pen_holder_tab_front_scoop/harness.ts b/eval/tasks/muse-pen_holder_tab_front_scoop/harness.ts new file mode 100644 index 000000000..a5b1dd507 --- /dev/null +++ b/eval/tasks/muse-pen_holder_tab_front_scoop/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-pen_holder_tab_front_scoop/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'pen_holder_tab_front_scoop' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-pen_holder_tab_front_scoop'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-pen_holder_tab_front_scoop/prompt.md b/eval/tasks/muse-pen_holder_tab_front_scoop/prompt.md new file mode 100644 index 000000000..05b060b41 --- /dev/null +++ b/eval/tasks/muse-pen_holder_tab_front_scoop/prompt.md @@ -0,0 +1,121 @@ +# pen_holder_tab_front_scoop (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a slot-assembled pen holder with a front access scoop, designed for organizing desktop stationery using interlocking flat panels. + +## Geometry and Dimensions +Approx. 92.0 mm × 82.0 mm × 122.0 mm. + +## Material +Timber + +## Manufacturing Method +Laser Cutting + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Desktop storage, holding pens and stationery items. + +## Structural Features +Base panel; front panel with access scoop; back panel; left panel; right panel; top frame. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +6 + +## Component Names +- base_panel +- front_panel +- back_panel +- left_panel +- right_panel +- top_frame + +## Adjustable Parameters +- **outer_width**: 92.0 (76.0 ~ 116.0 mm). Defines the overall width of the pen holder. +- **outer_depth**: 82.0 (66.0 ~ 106.0 mm). Defines the overall depth of the pen holder. +- **wall_height**: 110.0 (82.0 ~ 150.0 mm). Determines the internal storage height for pens and tools. +- **wall_thickness**: 5.0 (3.8 ~ 7.0 mm). Thickness of the vertical panels, ensuring structural rigidity and compatible with standard sheet materials. +- **base_thickness**: 6.0 (4.8 ~ 8.0 mm). Thickness of the bottom panel to provide a stable, load-bearing foundation. +- **frame_thickness**: 6.0 (4.8 ~ 8.0 mm). Thickness of the top frame used to lock the vertical walls together. +- **tab_width**: 18.0 (18.0 ~ 42.0 mm). Width of the interlocking tabs (tenons) for assembly. +- **frame_border**: 11.0 (1.0 ~ 25.0 mm). Width of the top frame's border, defining the top opening size. +- **front_notch_width**: 34.0 (18.0 ~ 58.0 mm). Width of the front access scoop for easy retrieval of shorter items. +- **front_notch_height**: 26.0 (50.0 ~ 66.0 mm). Height of the front access scoop. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. base_panel +The foundational support of the pen holder. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (mortise slots) for the four vertical panels. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = 0. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Features perimeter slots to receive the bottom tabs of the side panels. + +### 2. front_panel +The front enclosure of the pen holder. +* **Component Purpose**: Provides front containment and features a central scoop (notch) to allow easy access to shorter stationery items. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features tabs inserted into the base panel; top features tabs inserted into the top frame. + +### 3. back_panel +The rear enclosure of the pen holder. +* **Component Purpose**: Provides rear vertical support and containment. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features tabs inserted into the base panel; top features tabs inserted into the top frame. + +### 4. left_panel +The left side enclosure of the pen holder. +* **Component Purpose**: Provides lateral containment and structural rigidity along the Y-axis. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features tabs inserted into the base panel; top features tabs inserted into the top frame. + +### 5. right_panel +The right side enclosure of the pen holder. +* **Component Purpose**: Provides lateral containment and structural rigidity along the Y-axis. +* **Assembly Direction**: Inserted downwards along the -Z axis into the base panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Bottom features tabs inserted into the base panel; top features tabs inserted into the top frame. + +### 6. top_frame +The upper structural reinforcement of the pen holder. +* **Component Purpose**: Locks the four vertical panels together at the top, preventing outward deflection and providing a finished upper edge. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the assembled vertical panels. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Features perimeter slots that receive the top tabs of the front, back, left, and right panels. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 6-component model: + +* **front_panel -> base_panel** | Joint: interlocking | Note: Front panel bottom tabs inserted into base panel's front slots. +* **back_panel -> base_panel** | Joint: interlocking | Note: Back panel bottom tabs inserted into base panel's rear slots. +* **left_panel -> base_panel** | Joint: interlocking | Note: Left panel bottom tabs inserted into base panel's left slots. +* **right_panel -> base_panel** | Joint: interlocking | Note: Right panel bottom tabs inserted into base panel's right slots. +* **top_frame -> front_panel** | Joint: interlocking | Note: Top frame front slots pressed onto front panel's top tabs. +* **top_frame -> back_panel** | Joint: interlocking | Note: Top frame rear slots pressed onto back panel's top tabs. +* **top_frame -> left_panel** | Joint: interlocking | Note: Top frame left slots pressed onto left panel's top tabs. +* **top_frame -> right_panel** | Joint: interlocking | Note: Top frame right slots pressed onto right panel's top tabs. diff --git a/eval/tasks/muse-plant_shelf/harness.ts b/eval/tasks/muse-plant_shelf/harness.ts new file mode 100644 index 000000000..f7702f2a2 --- /dev/null +++ b/eval/tasks/muse-plant_shelf/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-plant_shelf/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'plant_shelf' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-plant_shelf'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-plant_shelf/prompt.md b/eval/tasks/muse-plant_shelf/prompt.md new file mode 100644 index 000000000..6fd3a06d5 --- /dev/null +++ b/eval/tasks/muse-plant_shelf/prompt.md @@ -0,0 +1,108 @@ +# plant_shelf (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a multi-tier wooden plant shelf with slatted shelves and dowel-based assembly for displaying potted plants. + +## Geometry and Dimensions +Approx. 630.0 mm × 310.0 mm × 750.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Load-bearing storage for plants. + +## Structural Features +Four vertical uprights; horizontal side rails (bottom and 3 tiers); slatted shelf panels for 3 tiers. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +36 + +## Component Names +- right_front_upright +- right_back_upright +- left_front_upright +- left_back_upright +- right_bottom_rail +- left_bottom_rail +- tier1_left_rail +- tier1_right_rail +- tier1_slat_01 to tier1_slat_08 +- tier2_left_rail +- tier2_right_rail +- tier2_slat_01 to tier2_slat_08 +- tier3_left_rail +- tier3_right_rail +- tier3_slat_01 to tier3_slat_08 + +## Adjustable Parameters +- **frame_width**: 600.0 (400.0 ~ 800.0 mm). Controls the overall width of the shelf frame. +- **frame_depth**: 300.0 (200.0 ~ 400.0 mm). Controls the depth of the shelf frame. +- **tier1_height**: 250.0 (150.0 ~ 400.0 mm). Sets the vertical position of the first shelf tier. +- **tier2_height**: 500.0 (350.0 ~ 650.0 mm). Sets the vertical position of the second shelf tier. +- **tier3_height**: 750.0 (550.0 ~ 900.0 mm). Sets the vertical position of the top shelf tier and determines the total height of the uprights. +- **board_thickness**: 10.0 (6.0 ~ 14.0 mm). Determines the structural thickness of the timber boards. +- **board_width**: 30.0 (20.0 ~ 40.0 mm). Determines the width of the uprights, rails, and slats. +- **slat_gap**: 7.0 (3.0 ~ 12.0 mm). Controls the spacing between adjacent slats on each tier for drainage and aesthetics. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Defines the radius of the dowel holes for assembly. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Determines the insertion depth of the dowel pins into the components. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~4. Uprights (Right Front, Right Back, Left Front, Left Back) +The vertical supporting entities of the shelf. +* **Component Purpose**: Vertical support. Transfers the load of the shelves to the ground, ensuring structural stability. +* **Assembly Direction**: Vertical base components, positioned along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features horizontal holes on the inner faces to connect with the side rails at each tier height. + +### 5~12. Rails (Bottom and Tiers 1-3, Left and Right) +The horizontal structural supports connecting the uprights. +* **Component Purpose**: Horizontal framework. Connects the front and back uprights and provides a resting base for the slats. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features holes on the ends for upright connection and on the top face for slat alignment. + +### 13~36. Slats (Tiers 1-3) +The functional shelf surfaces. +* **Component Purpose**: Shelf platform. Spans across the left and right rails to form the load-bearing surface for potted plants. +* **Assembly Direction**: Placed downwards along the -Z axis onto the tier rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features through-holes at each end to align with the top holes of the rails. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 36-component model: + +* **Left Bottom Rail -> Left Front/Back Uprights** | Joint: Dowel Joint | Note: Rail ends connect to upright inner faces. +* **Right Bottom Rail -> Right Front/Back Uprights** | Joint: Dowel Joint | Note: Rail ends connect to upright inner faces. +* **Tier Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to upright inner faces at respective tier heights. +* **Tier Slats -> Tier Rails** | Joint: Dowel Joint | Note: Slat bottom faces connect to rail top faces via dowel alignment. diff --git a/eval/tasks/muse-shoe_rack/harness.ts b/eval/tasks/muse-shoe_rack/harness.ts new file mode 100644 index 000000000..931b5c301 --- /dev/null +++ b/eval/tasks/muse-shoe_rack/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-shoe_rack/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'shoe_rack' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-shoe_rack'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-shoe_rack/prompt.md b/eval/tasks/muse-shoe_rack/prompt.md new file mode 100644 index 000000000..40b8a8010 --- /dev/null +++ b/eval/tasks/muse-shoe_rack/prompt.md @@ -0,0 +1,121 @@ +# shoe_rack (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a multi-tier, open-shelf shoe rack designed for wood-based assembly using dowel joints. + +## Geometry and Dimensions +Approx. 600.0 mm × 280.0 mm × 500.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Load-bearing storage for footwear. + +## Structural Features +Four vertical uprights; horizontal side rails (bottom, top, and intermediate tiers); horizontal slats forming the shelves. + +## Special Requirements +Keep assembly split unchanged. Ensure all dowel holes align perfectly between mating components. + +## Planned Component Quantity +38 + +## Component Names +- right_front_upright +- right_back_upright +- left_front_upright +- left_back_upright +- right_bottom_rail +- left_bottom_rail +- tier_1_left_rail +- tier_1_right_rail +- tier_1_slat_01 to tier_1_slat_08 +- tier_2_left_rail +- tier_2_right_rail +- tier_2_slat_01 to tier_2_slat_08 +- tier_3_left_rail +- tier_3_right_rail +- tier_3_slat_01 to tier_3_slat_08 +- right_top_rail +- left_top_rail + +## Adjustable Parameters +- **rack_width**: 600.0 (400.0 ~ 900.0 mm). Controls the overall width of the shoe rack to accommodate different spatial constraints. +- **rack_depth**: 280.0 (200.0 ~ 350.0 mm). Determines the depth of the shelves, sized to fit standard footwear. +- **rack_height**: 500.0 (350.0 ~ 700.0 mm). Controls the total height of the rack. +- **num_tiers**: 3 (2.0 ~ 5.0). Determines the number of storage shelves available. +- **board_thickness**: 10.0 (6.0 ~ 14.0 mm). Ensures structural stiffness of the timber boards while preventing excessive weight. +- **board_width**: 25.0 (15.0 ~ 40.0 mm). Defines the width of the individual structural members (uprights, rails, and slats). +- **slat_gap**: 8.0 (3.0 ~ 15.0 mm). Spacing between slats to provide ventilation for shoes and reduce material usage. +- **hole_radius**: 1.0 (0.5 ~ 3.0 mm). Radius of the dowel holes used for alignment and assembly. +- **hole_depth**: 3.0 (1.0 ~ 8.0 mm). Depth of the blind holes for dowel insertion, ensuring adequate bite without piercing through the boards. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~4. Uprights (Right Front, Right Back, Left Front, Left Back) +The primary vertical supports of the rack. +* **Component Purpose**: Vertical structural support. Transfers the load of the tiers to the ground and provides localization references for all horizontal rails. +* **Assembly Direction**: Vertical base components, positioned along the Z-axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features through-holes along the Y-axis for side rail dowels, and blind holes on the top face for top rail dowels. + +### 5~6. Bottom Rails (Right, Left) +The foundational horizontal links. +* **Component Purpose**: Structural linking between front and back uprights at the base to prevent splaying and ensure frame rigidity. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features dowel holes on the front and back ends mating with the uprights. + +### 7~12. Tier Rails (Left and Right for Tiers 1~3) +The horizontal supports for the shoe shelves. +* **Component Purpose**: Links the front and back uprights at each tier level and provides the mounting base for the horizontal slats. +* **Assembly Direction**: Inserted horizontally along the Y-axis between the front and back uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features dowel holes on the ends for upright connection, and upward-facing blind holes for slat alignment. + +### 13~36. Tier Slats (Tiers 1~3, Slats 01~08) +The resting surfaces for the footwear. +* **Component Purpose**: Spans the width of the rack to form the breathable shelf surfaces for storing shoes. +* **Assembly Direction**: Placed downwards along the -Z axis onto the tier rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features through-holes at each end that align with the upward-facing holes of the tier rails. + +### 37~38. Top Rails (Right, Left) +The upper capping structures. +* **Component Purpose**: Caps the front and back uprights, providing top structural rigidity and a finished look. +* **Assembly Direction**: Pressed downwards along the -Z axis onto the top faces of the uprights. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features vertical through-holes that drop dowels into the uprights' top blind holes. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 38-component model: + +* **Bottom Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to the lower side holes of the front and back uprights. +* **Tier Rails -> Uprights** | Joint: Dowel Joint | Note: Rail ends connect to the side holes of the uprights at their respective tier heights. +* **Top Rails -> Uprights** | Joint: Dowel Joint | Note: Top rail bottom faces connect to the upright top faces via vertical dowels. +* **Tier Slats -> Tier Rails** | Joint: Dowel Joint | Note: Slat ends connect to the upward-facing dowel holes on the left and right tier rails. diff --git a/eval/tasks/muse-stool_bar_round/harness.ts b/eval/tasks/muse-stool_bar_round/harness.ts new file mode 100644 index 000000000..6d7e3a5ec --- /dev/null +++ b/eval/tasks/muse-stool_bar_round/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_bar_round/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_bar_round' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_bar_round'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_bar_round/prompt.md b/eval/tasks/muse-stool_bar_round/prompt.md new file mode 100644 index 000000000..10a03f916 --- /dev/null +++ b/eval/tasks/muse-stool_bar_round/prompt.md @@ -0,0 +1,106 @@ +# stool_bar_round (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a tall round bar stool with mortise-and-tenon footrest rails designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 320.0 mm × 320.0 mm × 760.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating (elevated bar stool posture) with footrest load-bearing requirements. + +## Structural Features +Round seat panel; four tapered round legs; four footrest stretchers. + +## Special Requirements +Keep assembly split unchanged. The exported STEP must remain a closed solid. + +## Planned Component Quantity +9 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Ensures adequate load-bearing capacity and provides sufficient depth for the leg tenon sockets. +- **leg_height**: 742.0 (622.0 ~ 902.0 mm). Determines the overall seating height, strictly following ergonomic standards for bar stools. +- **tenon_height**: 9.0 (6.0 ~ 13.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_radius**: 160.0 (120.0 ~ 205.0 mm). Defines the seating area; constrained to prevent tipping caused by an unbalanced top-heavy ratio. +- **leg_top_radius**: 11.0 (8.0 ~ 16.0 mm). Defines the upper thickness of the leg, ensuring enough material to form the top tenon. +- **leg_bottom_radius**: 14.0 (10.0 ~ 20.0 mm). Defines the base footprint of the leg, providing anti-overturning stability. +- **tenon_radius**: 5.0 (4.0 ~ 8.0 mm). Controls the thickness of the connection joint to prevent shear failure. +- **stretcher_z**: 304.0 (254.0 ~ 384.0 mm). Sets the vertical placement of the footrest rails for ergonomic comfort and structural bracing. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Ensures the footrest can withstand vertical stepping loads. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Provides horizontal stiffness to the stretcher rails. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub and top surface of the stool. +* **Component Purpose**: Acts as the main load-bearing base for seating and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom face features four circular sockets arranged radially. + +### 2~5. Four Legs (leg_01, leg_02, leg_03, leg_04) +The primary vertical supporting entities of the stool. +* **Component Purpose**: Transfers the seat load to the ground. Features a tapered round profile for aesthetics and stability. Contains side mortises to receive the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). The top features a round tenon that fits into the seat panel. The sides feature angled mortises to connect with the stretchers. + +### 6~9. Four Stretchers (stretcher_01, stretcher_02, stretcher_03, stretcher_04) +The horizontal bracing and footrest entities. +* **Component Purpose**: Connects the legs together to prevent splaying under load and provides an ergonomic resting place for the user's feet. +* **Assembly Direction**: Inserted horizontally between the legs at height `stretcher_z`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature oriented tenons that insert into the side mortises of the corresponding legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket 1. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket 2. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket 3. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket 4. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg 1 and leg 2. +* **stretcher_02 -> leg_02 & leg_03** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg 2 and leg 3. +* **stretcher_03 -> leg_03 & leg_04** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg 3 and leg 4. +* **stretcher_04 -> leg_04 & leg_01** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg 4 and leg 1. diff --git a/eval/tasks/muse-stool_bar_square/harness.ts b/eval/tasks/muse-stool_bar_square/harness.ts new file mode 100644 index 000000000..5d509e196 --- /dev/null +++ b/eval/tasks/muse-stool_bar_square/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_bar_square/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_bar_square' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_bar_square'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_bar_square/prompt.md b/eval/tasks/muse-stool_bar_square/prompt.md new file mode 100644 index 000000000..89a7e09dd --- /dev/null +++ b/eval/tasks/muse-stool_bar_square/prompt.md @@ -0,0 +1,111 @@ +# stool_bar_square (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a tall square bar stool with apron rails and a lower footrest loop designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 312.0 mm × 312.0 mm × 738.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating (bar stool). + +## Structural Features +Seat panel; four legs; eight stretchers (forming an upper apron and a lower footrest loop). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +13 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 +- stretcher_05 +- stretcher_06 +- stretcher_07 +- stretcher_08 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Ensures structural integrity of the seat and provides sufficient depth for the leg tenons. +- **leg_height**: 720.0 (600.0 ~ 880.0 mm). Determines the seating height, strictly following ergonomic standards for bar counters. +- **tenon_height**: 9.0 (6.0 ~ 13.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_width**: 312.0 (222.0 ~ 402.0 mm). Defines the seating area and overall footprint width. +- **seat_depth**: 312.0 (222.0 ~ 402.0 mm). Defines the seating area and overall footprint depth. +- **leg_top_size**: 24.0 (18.0 ~ 34.0 mm). Controls the upper cross-section of the leg to ensure a flush fit and adequate material for the top tenon. +- **leg_bottom_size**: 28.0 (20.0 ~ 40.0 mm). Controls the base footprint of the leg, providing anti-overturning stability. +- **tenon_size**: 9.0 (6.0 ~ 13.0 mm). Ensures the tenon is robust enough to handle shear forces without weakening the leg top. +- **stretcher_z**: 288.0 (238.0 ~ 368.0 mm). Sets the height of the lower stretcher loop, functioning as a footrest. +- **stretcher_secondary_z**: 620.0 (560.0 ~ 700.0 mm). Sets the height of the upper stretcher loop, functioning as an apron rail for structural rigidity. +- **stretcher_bar_width**: 16.0 (12.0 ~ 22.0 mm). Determines the vertical stiffness of the horizontal supports. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the horizontal supports. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub and top surface of the stool. +* **Component Purpose**: Acts as the main load-bearing base for seating and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four square sockets to receive the leg tenons. + +### 2~5. leg_01 to leg_04 +The vertical supporting entities of the stool. +* **Component Purpose**: Transfers the seat load to the ground, ensuring stability. Features mortises on the sides to receive the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of length `tenon_height` that interference-fits into the bottom sockets of the seat panel. Sides feature mortises for stretcher connections. + +### 6~13. stretcher_01 to stretcher_08 +The horizontal bracing entities of the stool. +* **Component Purpose**: Connects the legs to prevent splaying, providing lateral stability. Arranged in a double-cycle configuration (lower footrest and upper apron). +* **Assembly Direction**: Inserted horizontally into the side mortises of the legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding leg mortises. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 13-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's bottom socket. +* **stretcher_01 to stretcher_04 -> leg_01 to leg_04** | Joint: interlocking | Note: Lower cycle stretchers inserted into leg side mortises at `stretcher_z`. +* **stretcher_05 to stretcher_08 -> leg_01 to leg_04** | Joint: interlocking | Note: Upper cycle stretchers inserted into leg side mortises at `stretcher_secondary_z`. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_box_brace/harness.ts b/eval/tasks/muse-stool_box_brace/harness.ts new file mode 100644 index 000000000..2ba7df300 --- /dev/null +++ b/eval/tasks/muse-stool_box_brace/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_box_brace/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_box_brace' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_box_brace'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_box_brace/prompt.md b/eval/tasks/muse-stool_box_brace/prompt.md new file mode 100644 index 000000000..4c4e1e355 --- /dev/null +++ b/eval/tasks/muse-stool_box_brace/prompt.md @@ -0,0 +1,79 @@ +# stool_box_brace (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a square stool with a clean lower box stretcher frame designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 326.0 mm × 326.0 mm × 444.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four legs; four stretchers forming a lower box frame. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Ensures adequate load-bearing capacity for the seat while accommodating the insertion depth of the leg tenons. +- **leg_height**: 428.0 (308.0 ~ 588.0 mm). Determines the seating height, strictly following ergonomic standards for seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_width**: 326.0 (236.0 ~ 416.0 mm). Defines the primary seating area width. +- **seat_depth**: 326.0 (236.0 ~ 416.0 mm). Defines the primary seating area depth. +- **leg_top_size**: 24.0 (18.0 ~ 34.0 mm). Controls the thickness of the leg at the top interface to ensure joint strength. +- **leg_bottom_size**: 28.0 (20.0 ~ 40.0 mm). Controls the footprint and base stability of the stool to prevent tipping. +- **tenon_size**: 9.0 (6.0 ~ 13.0 mm). Defines the cross-sectional size of the joint for structural integrity. +- **stretcher_z**: 138.0 (88.0 ~ 218.0 mm). Sets the vertical position of the stretcher frame to optimize leg stabilization and prevent splay. +- **stretcher_bar_width**: 14.0 (10.0 ~ 20.0 mm). Determines the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base for diff --git a/eval/tasks/muse-stool_cross_brace/harness.ts b/eval/tasks/muse-stool_cross_brace/harness.ts new file mode 100644 index 000000000..e7a79a64b --- /dev/null +++ b/eval/tasks/muse-stool_cross_brace/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_cross_brace/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_cross_brace' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_cross_brace'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_cross_brace/prompt.md b/eval/tasks/muse-stool_cross_brace/prompt.md new file mode 100644 index 000000000..c3450bec2 --- /dev/null +++ b/eval/tasks/muse-stool_cross_brace/prompt.md @@ -0,0 +1,104 @@ +# stool_cross_brace (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a square stool with crossing diagonal braces designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 330.0 mm × 330.0 mm × 456.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four legs; two crossing diagonal stretchers (braces). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Determines the structural strength of the seat and the maximum depth for leg tenons. +- **leg_height**: 440.0 (320.0 ~ 600.0 mm). Sets the seating height according to ergonomic standards. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the insertion depth of the leg tenons into the seat panel. +- **seat_width**: 330.0 (240.0 ~ 420.0 mm). Defines the lateral seating area. +- **seat_depth**: 330.0 (240.0 ~ 420.0 mm). Defines the longitudinal seating area. +- **leg_top_size**: 24.0 (18.0 ~ 34.0 mm). Controls the thickness of the leg at the top connection point. +- **leg_bottom_size**: 28.0 (20.0 ~ 40.0 mm). Controls the thickness of the leg at the floor contact point for stability. +- **tenon_size**: 9.0 (6.0 ~ 13.0 mm). Defines the cross-sectional size of the connecting tenons. +- **stretcher_z**: 182.0 (132.0 ~ 262.0 mm). Sets the vertical position of the cross braces to optimize structural rigidity and prevent leg splay. +- **stretcher_bar_width**: 16.0 (12.0 ~ 22.0 mm). Determines the vertical profile width of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal thickness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets to receive the leg tenons. + +### 2~5. Four Legs (leg_01, leg_02, leg_03, leg_04) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Features side mortises to receive the diagonal stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. Mid-section features angled mortises for the stretchers. + +### 6~7. Stretchers (stretcher_01, stretcher_02) +The diagonal bracing entities of the stool. +* **Component Purpose**: Structural reinforcement. Connects opposite diagonal legs to form an 'X' brace, preventing leg splay and increasing overall torsional rigidity. +* **Assembly Direction**: Inserted horizontally/diagonally between the respective leg pairs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding side mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **stretcher_01 -> leg_01 & leg_04** | Joint: interlocking | Note: Stretcher ends inserted into side mortises on leg_01 and leg_04. +* **stretcher_02 -> leg_02 & leg_03** | Joint: interlocking | Note: Stretcher ends inserted into side mortises on leg_02 and leg_03. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_hex/harness.ts b/eval/tasks/muse-stool_hex/harness.ts new file mode 100644 index 000000000..e7f10ee63 --- /dev/null +++ b/eval/tasks/muse-stool_hex/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_hex/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_hex' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_hex'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_hex/prompt.md b/eval/tasks/muse-stool_hex/prompt.md new file mode 100644 index 000000000..826c9e330 --- /dev/null +++ b/eval/tasks/muse-stool_hex/prompt.md @@ -0,0 +1,107 @@ +# stool_hex (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a hexagonal stool with four legs and a lower stretcher loop designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 368.0 mm × 368.0 mm × 454.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Hexagonal seat panel; four legs; four lower stretchers forming a reinforcing loop. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Ensures sufficient structural strength for seating while accommodating the insertion depth of the leg tenons. +- **leg_height**: 438.0 (318.0 ~ 598.0 mm). Determines the ergonomic seating height of the stool. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Controls the bite depth of the physical connections between the legs and the seat panel. +- **seat_polygon_radius**: 184.0 (144.0 ~ 229.0 mm). Defines the overall size and seating area of the hexagonal seat panel. +- **leg_top_size**: 25.0 (18.0 ~ 35.0 mm). Determines the thickness of the leg at the connection point with the seat, balancing aesthetics and joint strength. +- **leg_bottom_size**: 29.0 (21.0 ~ 41.0 mm). Determines the footprint thickness of the leg to ensure ground stability and prevent tipping. +- **tenon_size**: 9.0 (6.0 ~ 13.0 mm). Defines the cross-sectional size of the joint to balance leg strength and seat integrity. +- **stretcher_z**: 166.0 (116.0 ~ 246.0 mm). Sets the vertical position of the stretcher loop for optimal structural bracing and footrest ergonomics. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Defines the vertical stiffness and load-bearing capacity of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Defines the horizontal stiffness of the stretcher bars to resist lateral forces. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned horizontally at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four square sockets to receive the leg tenons. + +### 2~5. leg_01 to leg_04 +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Provides side mortises to anchor the stretcher loop. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a square tenon that interference-fits into the bottom sockets of the seat panel. Sides feature mortises to receive the stretcher tenons. + +### 6~9. stretcher_01 to stretcher_04 +The lateral bracing entities of the stool. +* **Component Purpose**: Horizontal support. Connects the legs together to form a rigid lower loop, preventing leg splay under load and increasing overall structural integrity. +* **Assembly Direction**: Inserted horizontally between adjacent legs at $Z = stretcher\_z$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding side mortises of the adjacent legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding bottom socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding bottom socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding bottom socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding bottom socket. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_02 -> leg_02 & leg_04** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_03 -> leg_04 & leg_03** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_04 -> leg_03 & leg_01** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_hex_bar/harness.ts b/eval/tasks/muse-stool_hex_bar/harness.ts new file mode 100644 index 000000000..33d053a8d --- /dev/null +++ b/eval/tasks/muse-stool_hex_bar/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_hex_bar/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_hex_bar' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_hex_bar'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_hex_bar/prompt.md b/eval/tasks/muse-stool_hex_bar/prompt.md new file mode 100644 index 000000000..8e90685af --- /dev/null +++ b/eval/tasks/muse-stool_hex_bar/prompt.md @@ -0,0 +1,110 @@ +# stool_hex_bar (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a tall bar stool with a hexagonal seat and dual loops of stretchers, designed for wood-based assembly using mortise-and-tenon joints. + +## Geometry and Dimensions +Approx. 344.0 mm × 344.0 mm × 720.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating (bar/tall stool). + +## Structural Features +Hexagonal seat panel; four legs; eight stretchers (forming dual reinforcement loops). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +13 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 +- Stretcher 05 +- Stretcher 06 +- Stretcher 07 +- Stretcher 08 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Ensures adequate load-bearing capacity for the user and sufficient depth for leg tenon insertion. +- **leg_height**: 702.0 (582.0 ~ 862.0 mm). Determines the overall seating height, suitable for bar or counter-height applications. +- **tenon_height**: 9.0 (6.0 ~ 13.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_polygon_radius**: 172.0 (132.0 ~ 217.0 mm). Defines the seating area; constrained to prevent tipping caused by an unbalanced base-to-seat ratio. +- **leg_top_size**: 24.0 (18.0 ~ 34.0 mm). Controls the thickness of the leg at the top interface, ensuring sufficient material around the tenon. +- **leg_bottom_size**: 30.0 (22.0 ~ 42.0 mm). Controls the footprint thickness of the leg, providing a stable base and lowering the center of gravity. +- **tenon_size**: 9.0 (6.0 ~ 13.0 mm). Defines the cross-sectional strength of the joint connecting the legs to the seat. +- **stretcher_z**: 284.0 (234.0 ~ 364.0 mm). Sets the height of the lower stretcher loop, acting as a structural tie and an ergonomic footrest. +- **stretcher_secondary_z**: 596.0 (536.0 ~ 676.0 mm). Sets the height of the upper stretcher loop, providing critical anti-splay reinforcement near the top of the tall legs. +- **stretcher_bar_width**: 14.0 (10.0 ~ 20.0 mm). Determines the vertical stiffness of the stretchers. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the stretchers and limits the mortise depth required in the legs. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub and seating surface of the stool. +* **Component Purpose**: Acts as the main load-bearing base for the user and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four square sockets to receive the leg tenons. + +### 2~5. Four Legs (Leg 01, Leg 02, Leg 03, Leg 04) +The primary vertical supporting entities of the stool. +* **Component Purpose**: Transfers the seat load to the ground. Tapered design (wider at the bottom) ensures anti-overturning stability. Contains mortises to receive stretcher tenons. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a square tenon that fits into the seat panel. Side faces feature mortises to receive the stretchers. + +### 6~13. Eight Stretchers (Stretcher 01 to Stretcher 08) +The horizontal bracing entities forming dual structural loops. +* **Component Purpose**: Prevents the tall legs from splaying under load, significantly increasing the rigidity of the frame. The lower loop also functions as a footrest. +* **Assembly Direction**: Inserted horizontally between adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 13-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **Stretcher 01~04 -> Legs** | Joint: interlocking | Note: Lower loop stretchers inserted into the lower mortises of adjacent legs. +* **Stretcher 05~08 -> Legs** | Joint: interlocking | Note: Upper loop stretchers inserted into the upper mortises of adjacent legs. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; leg connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_inset_legs/harness.ts b/eval/tasks/muse-stool_inset_legs/harness.ts new file mode 100644 index 000000000..2eddbdfb7 --- /dev/null +++ b/eval/tasks/muse-stool_inset_legs/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_inset_legs/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_inset_legs' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_inset_legs'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_inset_legs/prompt.md b/eval/tasks/muse-stool_inset_legs/prompt.md new file mode 100644 index 000000000..e1042d83e --- /dev/null +++ b/eval/tasks/muse-stool_inset_legs/prompt.md @@ -0,0 +1,108 @@ +# stool_inset_legs (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged inset stool with a higher apron loop (stretchers) designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 320.0 mm × 320.0 mm × 436.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four legs; four stretchers (apron loop). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 + +## Adjustable Parameters +- **seat_width**: 320.0 (230.0 ~ 410.0 mm). Determines the overall width of the seating area. +- **seat_depth**: 320.0 (230.0 ~ 410.0 mm). Determines the overall depth of the seating area. +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Ensures adequate load-bearing capacity for the seat and provides sufficient depth for the leg tenons. +- **leg_height**: 420.0 (300.0 ~ 580.0 mm). Determines the seating height, strictly following ergonomic standards for single-person seating posture. +- **leg_top_size**: 28.0 (20.0 ~ 38.0 mm). Controls the thickness of the leg at the top connection point to ensure joint integrity. +- **leg_bottom_size**: 30.0 (22.0 ~ 42.0 mm). Controls the thickness of the leg at the base to ensure anti-overturning stability. +- **tenon_size**: 10.0 (7.0 ~ 14.0 mm). Determines the cross-sectional size of the tenon for the leg-to-seat joint. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the insertion bite depth of the leg tenons into the seat panel. +- **stretcher_z**: 236.0 (186.0 ~ 316.0 mm). Sets the vertical position of the stretcher loop to prevent leg splay and provide structural rigidity. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Controls the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Controls the horizontal stiffness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four rectangular sockets generated via boolean cut. + +### 2~5. Four Legs (Leg 01, Leg 02, Leg 03, Leg 04) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring stability in the X-Y plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). The top features a tenon of height `tenon_height` that fits into the bottom sockets of the seat panel. The sides feature mortises to receive the stretcher tenons. + +### 6~9. Four Stretchers (Stretcher 01, Stretcher 02, Stretcher 03, Stretcher 04) +The horizontal bracing entities of the stool. +* **Component Purpose**: Forms an apron loop to prevent leg splay and significantly increases the overall structural rigidity of the stool frame. +* **Assembly Direction**: Inserted horizontally into the side mortises of the adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends of each stretcher feature tenons that are inserted into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of Leg 01 and Leg 02. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of Leg 02 and Leg 03. +* **Stretcher 03 -> Leg 03 & Leg 04** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of Leg 03 and Leg 04. +* **Stretcher 04 -> Leg 04 & Leg 01** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of Leg 04 and Leg 01. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all vertical connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_low_round/harness.ts b/eval/tasks/muse-stool_low_round/harness.ts new file mode 100644 index 000000000..b4d784e45 --- /dev/null +++ b/eval/tasks/muse-stool_low_round/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_low_round/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_low_round' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_low_round'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_low_round/prompt.md b/eval/tasks/muse-stool_low_round/prompt.md new file mode 100644 index 000000000..466727807 --- /dev/null +++ b/eval/tasks/muse-stool_low_round/prompt.md @@ -0,0 +1,107 @@ +# stool_low_round (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a four-legged low round stool with a near-floor stretcher loop designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 372.0 mm × 372.0 mm × 320.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Round seat panel; four legs; four stretchers forming a stabilizing loop. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 + +## Adjustable Parameters +- **seat_thickness**: 20.0 (12.0 ~ 34.0 mm). Determines the load-bearing capacity of the seat and provides adequate depth for the leg tenons. +- **leg_height**: 300.0 (250.0 ~ 460.0 mm). Sets the overall seating height of the stool. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_radius**: 186.0 (146.0 ~ 231.0 mm). Defines the seating area and the overall width/depth footprint of the stool. +- **leg_top_size**: 30.0 (22.0 ~ 40.0 mm). Defines the structural thickness of the leg at the seat junction. +- **leg_bottom_size**: 34.0 (26.0 ~ 46.0 mm). Defines the structural thickness of the leg at the floor contact point, ensuring stability. +- **tenon_size**: 10.0 (7.0 ~ 14.0 mm). Controls the cross-sectional area of the leg tenons to balance joint strength and prevent breaking. +- **stretcher_z**: 106.0 (80.0 ~ 186.0 mm). Sets the vertical position of the stretcher loop from the floor to optimize anti-splaying leverage. +- **stretcher_bar_width**: 16.0 (12.0 ~ 22.0 mm). Defines the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Defines the horizontal stiffness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets to receive the leg tenons. + +### 2~5. Four Legs (Leg 01, Leg 02, Leg 03, Leg 04) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability in the X-Y plane. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. Sides feature mortises to receive the stretcher tenons. + +### 6~9. Four Stretchers (Stretcher 01, Stretcher 02, Stretcher 03, Stretcher 04) +The stabilizing loop of the stool. +* **Component Purpose**: Horizontal bracing. Connects the legs near the floor in a cyclic loop to prevent splaying and increase overall structural rigidity. +* **Assembly Direction**: Inserted horizontally into the side mortises of the legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature angled tenons that fit into the corresponding leg mortises. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **Stretcher 03 -> Leg 03 & Leg 04** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **Stretcher 04 -> Leg 04 & Leg 01** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_octagon/harness.ts b/eval/tasks/muse-stool_octagon/harness.ts new file mode 100644 index 000000000..9d1588ba4 --- /dev/null +++ b/eval/tasks/muse-stool_octagon/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_octagon/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_octagon' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_octagon'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_octagon/prompt.md b/eval/tasks/muse-stool_octagon/prompt.md new file mode 100644 index 000000000..907e1e967 --- /dev/null +++ b/eval/tasks/muse-stool_octagon/prompt.md @@ -0,0 +1,112 @@ +# stool_octagon (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct an octagonal stool with perimeter rails and diagonal braces designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 352.0 mm × 352.0 mm × 464.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Octagonal seat panel; four legs; six stretchers (four perimeter rails and two diagonal braces). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +11 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 +- stretcher_05 +- stretcher_06 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Must be thick enough to accommodate the insertion depth of the leg tenons and support the user's weight without bowing. +- **leg_height**: 448.0 (328.0 ~ 608.0 mm). Strictly follows ergonomic standards for single-person seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_polygon_radius**: 176.0 (136.0 ~ 221.0 mm). Defines the overall seating area; constrained to prevent tipping caused by an unbalanced base-to-seat ratio. +- **leg_top_size**: 22.0 (18.0 ~ 32.0 mm). Ensures sufficient material at the top of the leg to form a robust tenon. +- **leg_bottom_size**: 24.0 (20.0 ~ 36.0 mm). Provides a stable footprint and load-bearing stiffness at the base. +- **tenon_size**: 8.0 (6.0 ~ 12.0 mm). Controls the cross-sectional strength of the joint connecting the legs to the seat. +- **stretcher_z**: 162.0 (112.0 ~ 242.0 mm). Sets the height of the primary perimeter stretchers to prevent leg splay and provide structural rigidity. +- **stretcher_secondary_z**: 216.0 (156.0 ~ 296.0 mm). Sets the height of the secondary diagonal (X-brace) stretchers to avoid physical interference with the primary stretchers. +- **stretcher_bar_width**: 14.0 (10.0 ~ 20.0 mm). Ensures the stretcher has enough material to resist bending and torsion. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Provides adequate thickness for cutting tenons on the ends of the stretchers. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub and load-bearing surface of the stool. +* **Component Purpose**: Acts as the main seating surface and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets to receive the leg tenons. + +### 2~5. leg_01 to leg_04 +The vertical supporting entities of the stool. +* **Component Purpose**: Transfers the seat load to the ground, ensuring anti-overturning stability. Contains mortises on the sides to receive the stretcher tenons. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. Sides feature mortises for the stretchers. + +### 6~11. stretcher_01 to stretcher_06 +The horizontal and diagonal bracing entities of the stool. +* **Component Purpose**: Connects the legs together to prevent splaying, increasing the overall rigidity and structural integrity of the base. Configured in a "box_x" mode (four perimeter rails and two diagonal braces). +* **Assembly Direction**: Inserted horizontally/diagonally in the X-Y plane into the leg mortises. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 11-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Perimeter stretcher tenons inserted into adjacent leg mortises. +* **stretcher_02 -> leg_02 & leg_04** | Joint: interlocking | Note: Perimeter stretcher tenons inserted into adjacent leg mortises. +* **stretcher_03 -> leg_04 & leg_03** | Joint: interlocking | Note: Perimeter stretcher tenons inserted into adjacent leg mortises. +* **stretcher_04 -> leg_03 & leg_01** | Joint: interlocking | Note: Perimeter stretcher tenons inserted into adjacent leg mortises. +* **stretcher_05 -> leg_01 & leg_04** | Joint: interlocking | Note: Diagonal (X-brace) stretcher tenons inserted into opposite leg mortises. +* **stretcher_06 -> leg_02 & leg_03** | Joint: interlocking | Note: Diagonal (X-brace) stretcher tenons inserted into opposite leg mortises. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_oval_apron/harness.ts b/eval/tasks/muse-stool_oval_apron/harness.ts new file mode 100644 index 000000000..d61671601 --- /dev/null +++ b/eval/tasks/muse-stool_oval_apron/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_oval_apron/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_oval_apron' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_oval_apron'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_oval_apron/prompt.md b/eval/tasks/muse-stool_oval_apron/prompt.md new file mode 100644 index 000000000..84826c243 --- /dev/null +++ b/eval/tasks/muse-stool_oval_apron/prompt.md @@ -0,0 +1,108 @@ +# stool_oval_apron (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct an oval stool with a high apron-style stretcher loop designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 380.0 mm × 292.0 mm × 450.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Oval seat panel; four legs; four apron-style stretchers forming a continuous loop. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Must be thick enough to accommodate the insertion depth of the leg tenons and support the user's weight. +- **leg_height**: 432.0 (312.0 ~ 592.0 mm). Determines the overall seating height, strictly following ergonomic standards for seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_radius_x**: 190.0 (150.0 ~ 240.0 mm). Controls the primary width of the oval seat to ensure adequate seating area. +- **seat_radius_y**: 146.0 (111.0 ~ 191.0 mm). Controls the depth of the oval seat. +- **leg_top_size**: 26.0 (18.0 ~ 36.0 mm). Ensures sufficient material at the top of the leg for the tenon and structural integrity. +- **leg_bottom_size**: 30.0 (22.0 ~ 42.0 mm). Provides a stable base footprint to prevent tipping. +- **tenon_size**: 9.5 (6.5 ~ 13.5 mm). Controls the cross-sectional area of the tenon for optimal shear strength. +- **stretcher_z**: 280.0 (230.0 ~ 360.0 mm). Sets the vertical position of the apron-style stretchers to maximize leg stability and prevent splaying. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Determines the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Ensures the stretchers are robust enough to resist lateral forces without protruding too much. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets generated via boolean cut to receive the leg tenons. + +### 2~5. leg_01, leg_02, leg_03, leg_04 +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Features side mortises to receive the stretcher loop. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. Sides feature mortises for the stretcher tenons. + +### 6~9. stretcher_01, stretcher_02, stretcher_03, stretcher_04 +The horizontal bracing entities of the stool. +* **Component Purpose**: Horizontal support. Forms a high apron-style loop connecting the legs to prevent splaying and increase overall structural rigidity. +* **Assembly Direction**: Inserted horizontally into the side mortises of adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding leg mortises. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_02 -> leg_02 & leg_03** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_03 -> leg_03 & leg_04** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **stretcher_04 -> leg_04 & leg_01** | Joint: interlocking | Note: Stretcher ends inserted into side mortises of adjacent legs. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_pentagon_five_leg/harness.ts b/eval/tasks/muse-stool_pentagon_five_leg/harness.ts new file mode 100644 index 000000000..705108db6 --- /dev/null +++ b/eval/tasks/muse-stool_pentagon_five_leg/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_pentagon_five_leg/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_pentagon_five_leg' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_pentagon_five_leg'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_pentagon_five_leg/prompt.md b/eval/tasks/muse-stool_pentagon_five_leg/prompt.md new file mode 100644 index 000000000..3776c0d68 --- /dev/null +++ b/eval/tasks/muse-stool_pentagon_five_leg/prompt.md @@ -0,0 +1,111 @@ +# stool_pentagon_five_leg (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a five-leg pentagonal stool with a continuous mortise-and-tenon rail loop (stretcher) designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 364.0 mm × 364.0 mm × 463.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; five legs; five stretchers forming a continuous rail loop. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +11 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Leg 05 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 +- Stretcher 05 + +## Adjustable Parameters +- **seat_thickness**: 17.0 (10.0 ~ 31.0 mm). Determines the structural strength of the seat and the maximum depth for leg tenons. +- **leg_height**: 446.0 (326.0 ~ 606.0 mm). Sets the seating height, adhering to ergonomic standards for stools. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the leg-to-seat physical connections. +- **seat_polygon_radius**: 182.0 (142.0 ~ 227.0 mm). Defines the overall seating area and footprint of the stool. +- **leg_top_radius**: 11.5 (8.0 ~ 16.5 mm). Controls the thickness of the leg at the top connection point. +- **leg_bottom_radius**: 13.5 (10.0 ~ 19.5 mm). Controls the thickness of the leg at the base for ground contact stability. +- **tenon_radius**: 5.0 (4.0 ~ 8.0 mm). Sets the thickness of the cylindrical tenon connecting the leg to the seat. +- **stretcher_z**: 180.0 (130.0 ~ 260.0 mm). Defines the vertical placement of the stretcher loop for structural rigidity and footrest ergonomics. +- **stretcher_bar_width**: 14.0 (10.0 ~ 20.0 mm). Determines the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the five legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features five circular sockets arranged in a radial layout. + +### 2~6. Five Legs (Leg 01 to Leg 05) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability in the X-Y plane. Also provides mortises for the stretcher loop. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a cylindrical tenon of height `tenon_height` that interference-fits into the bottom sockets of the seat panel. The sides feature mortises to receive the stretcher tenons. + +### 7~11. Five Stretchers (Stretcher 01 to Stretcher 05) +The horizontal bracing entities of the stool. +* **Component Purpose**: Horizontal support. Connects the legs together to form a rigid continuous loop, preventing leg splay and increasing overall structural integrity under load. +* **Assembly Direction**: Inserted horizontally into the side mortises of adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that fit into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 11-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **Leg 05 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 5. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Connects adjacent legs to form the rail loop. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Connects adjacent legs to form the rail loop. +* **Stretcher 03 -> Leg 03 & Leg 04** | Joint: interlocking | Note: Connects adjacent legs to form the rail loop. +* **Stretcher 04 -> Leg 04 & Leg 05** | Joint: interlocking | Note: Connects adjacent legs to form the rail loop. +* **Stretcher 05 -> Leg 05 & Leg 01** | Joint: interlocking | Note: Connects adjacent legs to form the rail loop. +* **Seat Panel -> All Legs** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_rect_compact/harness.ts b/eval/tasks/muse-stool_rect_compact/harness.ts new file mode 100644 index 000000000..f0d49066f --- /dev/null +++ b/eval/tasks/muse-stool_rect_compact/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_rect_compact/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_rect_compact' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_rect_compact'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_rect_compact/prompt.md b/eval/tasks/muse-stool_rect_compact/prompt.md new file mode 100644 index 000000000..5c2f130b4 --- /dev/null +++ b/eval/tasks/muse-stool_rect_compact/prompt.md @@ -0,0 +1,108 @@ +# stool_rect_compact (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a compact rectangular utility stool with low rails (stretchers) designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 320.0 mm × 250.0 mm × 378.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating, utility step or resting stool. + +## Structural Features +Seat panel; four legs; four stretchers (low rails). + +## Special Requirements +Keep assembly split unchanged. The exported STEP must remain a closed solid. + +## Planned Component Quantity +9 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Must be thick enough to accommodate the insertion depth of the leg tenons and support seating loads. +- **leg_height**: 362.0 (250.0 ~ 522.0 mm). Determines the overall seating height, strictly following ergonomic standards for a utility stool. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_width**: 320.0 (230.0 ~ 410.0 mm). Constrains the extreme values to prevent tipping caused by unbalanced length-to-width ratios. +- **seat_depth**: 250.0 (220.0 ~ 340.0 mm). Provides adequate seating area while maintaining a compact footprint. +- **leg_top_size**: 26.0 (18.0 ~ 36.0 mm). Ensures sufficient material for the top tenon and structural connection to the seat. +- **leg_bottom_size**: 28.0 (20.0 ~ 40.0 mm). Lower limit ensures load-bearing stiffness and stability at the base. +- **tenon_size**: 9.5 (6.5 ~ 13.5 mm). Controls the thickness of the tenon to balance joint strength and prevent breaking. +- **stretcher_z**: 136.0 (86.0 ~ 216.0 mm). Sets the height of the low rails to optimize leg bracing and structural rigidity. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Ensures the stretcher rails can resist bending and buckling forces. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Provides enough material to form the stretcher tenons without weakening the rail. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets/mortises) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets to receive the leg tenons. + +### 2~5. Four Legs (leg_01, leg_02, leg_03, leg_04) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Contains mortises along the shaft to receive stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon that interference-fits into the bottom sockets of the seat panel. The sides feature mortises to connect with the stretchers. + +### 6~9. Four Stretchers (stretcher_01, stretcher_02, stretcher_03, stretcher_04) +The horizontal bracing entities of the stool. +* **Component Purpose**: Horizontal support. Connects the legs together to prevent splaying and increases the overall structural rigidity of the base. +* **Assembly Direction**: Inserted horizontally in the X-Y plane into the leg mortises. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg_01 and leg_02. +* **stretcher_02 -> leg_02 & leg_04** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg_02 and leg_04. +* **stretcher_03 -> leg_04 & leg_03** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg_04 and leg_03. +* **stretcher_04 -> leg_03 & leg_01** | Joint: interlocking | Note: Stretcher tenons inserted into side mortises of leg_03 and leg_01. +* **seat_panel -> All Leg Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_round_tripod/harness.ts b/eval/tasks/muse-stool_round_tripod/harness.ts new file mode 100644 index 000000000..653c01a57 --- /dev/null +++ b/eval/tasks/muse-stool_round_tripod/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_round_tripod/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_round_tripod' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_round_tripod'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_round_tripod/prompt.md b/eval/tasks/muse-stool_round_tripod/prompt.md new file mode 100644 index 000000000..f1bbe962d --- /dev/null +++ b/eval/tasks/muse-stool_round_tripod/prompt.md @@ -0,0 +1,103 @@ +# stool_round_tripod (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a round tripod stool with a triangular mortise-and-tenon brace designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 344.0 mm × 344.0 mm × 454.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Round seat panel; three legs; three stretchers forming a triangular brace. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Ensures adequate strength for the seat base while accommodating the insertion depth of the leg tenons. +- **leg_height**: 438.0 (318.0 ~ 598.0 mm). Determines the seating height, strictly following ergonomic standards for seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_radius**: 172.0 (132.0 ~ 217.0 mm). Defines the seating area and overall width of the stool. +- **leg_top_radius**: 13.0 (9.0 ~ 18.0 mm). Defines the upper thickness of the leg for structural support at the joint interface. +- **leg_bottom_radius**: 15.0 (11.0 ~ 21.0 mm). Defines the base footprint of the leg for ground stability. +- **tenon_radius**: 5.5 (4.0 ~ 8.5 mm). Controls the thickness of the top tenon to prevent breakage while fitting into the seat. +- **stretcher_z**: 164.0 (114.0 ~ 244.0 mm). Sets the vertical position of the stretcher brace for optimal structural rigidity and leg stabilization. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Determines the vertical stiffness of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the three legs. +* **Assembly Direction**: Fixed base component, positioned at absolute Z = `leg_height`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features three round sockets arranged radially. + +### 2~4. Legs (Leg 01, Leg 02, Leg 03) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability in a radial layout. Also provides mortises for the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a round tenon inserted into the seat panel. Mid-section features angled mortises for stretcher insertion. + +### 5~7. Stretchers (Stretcher 01, Stretcher 02, Stretcher 03) +The bracing entities of the stool. +* **Component Purpose**: Horizontal bracing. Connects the legs together to form a rigid triangular structure, preventing leg splay under load. +* **Assembly Direction**: Inserted horizontally/angularly between the adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding leg mortises. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's first radial socket. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's second radial socket. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's third radial socket. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Connects Leg 01 and Leg 02 via end tenons. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Connects Leg 02 and Leg 03 via end tenons. +* **Stretcher 03 -> Leg 03 & Leg 01** | Joint: interlocking | Note: Connects Leg 03 and Leg 01 via end tenons. +* **Seat Panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all top connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_splayed_round/harness.ts b/eval/tasks/muse-stool_splayed_round/harness.ts new file mode 100644 index 000000000..3cfdc4068 --- /dev/null +++ b/eval/tasks/muse-stool_splayed_round/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_splayed_round/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_splayed_round' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_splayed_round'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_splayed_round/prompt.md b/eval/tasks/muse-stool_splayed_round/prompt.md new file mode 100644 index 000000000..6f0a79802 --- /dev/null +++ b/eval/tasks/muse-stool_splayed_round/prompt.md @@ -0,0 +1,107 @@ +# stool_splayed_round (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a round stool with splayed round legs and a square stretcher loop designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 352.0 mm × 352.0 mm × 465.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Round seat panel; four splayed round legs; four rectangular stretchers forming a continuous loop. + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +9 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 + +## Adjustable Parameters +- **seat_thickness**: 17.0 (10.0 ~ 31.0 mm). Must be thick enough to accommodate the insertion depth of the leg tenons without breaking through the top surface. +- **leg_height**: 448.0 (328.0 ~ 608.0 mm). Determines the overall height of the stool, strictly following ergonomic standards for seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_radius**: 176.0 (136.0 ~ 221.0 mm). Defines the seating area; constrains the extreme values to prevent tipping caused by an unbalanced base-to-seat ratio. +- **leg_top_radius**: 12.0 (8.0 ~ 17.0 mm). Ensures sufficient material at the top of the leg to support the tenon and bear the vertical load. +- **leg_bottom_radius**: 15.0 (11.0 ~ 21.0 mm). Provides a stable footprint and structural stiffness at the base of the stool. +- **tenon_radius**: 5.5 (4.0 ~ 8.5 mm). Controls the thickness of the connecting tenon to balance joint strength and prevent snapping. +- **stretcher_z**: 176.0 (126.0 ~ 256.0 mm). Sets the vertical position of the stretcher loop to provide optimal bracing against leg splay under load. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Ensures the horizontal bracing members have adequate stiffness to resist bending. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Lower limit ensures load-bearing stiffness; upper limit prevents interference with the leg geometry. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. seat_panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (round sockets) for the four splayed legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four round sockets arranged radially. + +### 2~5. Four Legs (leg_01, leg_02, leg_03, leg_04) +The supporting entities of the stool. +* **Component Purpose**: Vertical and lateral support. Transfers the seat load to the ground, with a splayed angle ensuring anti-overturning stability in the X-Y plane. +* **Assembly Direction**: Inserted upwards along their respective splayed axes into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a round tenon of height `tenon_height` that interference-fits into the bottom sockets of the seat panel. The mid-section features angled mortises to receive the stretchers. + +### 6~9. Four Stretchers (stretcher_01, stretcher_02, stretcher_03, stretcher_04) +The horizontal bracing entities of the stool. +* **Component Purpose**: Structural reinforcement. Connects the four legs in a continuous square loop to prevent them from splaying outward under heavy vertical loads. +* **Assembly Direction**: Inserted horizontally between adjacent legs at height `stretcher_z`. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature rectangular tenons that insert into the corresponding mortises on the inner faces of the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 9-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **stretcher_01 -> leg_01 & leg_02** | Joint: interlocking | Note: Connects adjacent legs to form the first side of the square loop. +* **stretcher_02 -> leg_02 & leg_03** | Joint: interlocking | Note: Connects adjacent legs to form the second side of the square loop. +* **stretcher_03 -> leg_03 & leg_04** | Joint: interlocking | Note: Connects adjacent legs to form the third side of the square loop. +* **stretcher_04 -> leg_04 & leg_01** | Joint: interlocking | Note: Connects adjacent legs to close the square loop. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_splayed_square/harness.ts b/eval/tasks/muse-stool_splayed_square/harness.ts new file mode 100644 index 000000000..c19b20482 --- /dev/null +++ b/eval/tasks/muse-stool_splayed_square/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_splayed_square/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_splayed_square' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_splayed_square'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_splayed_square/prompt.md b/eval/tasks/muse-stool_splayed_square/prompt.md new file mode 100644 index 000000000..8fa239367 --- /dev/null +++ b/eval/tasks/muse-stool_splayed_square/prompt.md @@ -0,0 +1,104 @@ +# stool_splayed_square (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a square stool with splayed legs and X-stretchers designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 320.0 mm × 320.0 mm × 446.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four splayed legs; two intersecting stretchers (X-stretcher layout). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +7 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 + +## Adjustable Parameters +- **seat_thickness**: 16.0 (10.0 ~ 30.0 mm). Determines the structural strength of the seat and provides adequate depth for the leg tenon sockets. +- **leg_height**: 430.0 (310.0 ~ 590.0 mm). Sets the overall seating height, strictly following ergonomic standards for stools. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the insertion depth of the leg tenons into the seat panel. +- **seat_width**: 320.0 (230.0 ~ 410.0 mm). Defines the primary seating area width. +- **seat_depth**: 320.0 (230.0 ~ 410.0 mm). Defines the primary seating area depth. +- **leg_top_size**: 26.0 (18.0 ~ 36.0 mm). Defines the thickness of the leg at the top connection point to ensure joint stability. +- **leg_bottom_size**: 32.0 (24.0 ~ 44.0 mm). Defines the thickness of the leg at the floor contact point for load distribution. +- **tenon_size**: 9.5 (6.5 ~ 13.5 mm). Controls the cross-sectional size of the joint to balance tenon strength and mortise wall thickness. +- **stretcher_z**: 180.0 (130.0 ~ 260.0 mm). Sets the vertical position of the X-stretchers to optimize the bracing angle and leg clearance. +- **stretcher_bar_width**: 16.0 (12.0 ~ 22.0 mm). Ensures the cross-bracing has sufficient stiffness against bending. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Maintains the structural integrity of the stretcher while preventing interference at the X-intersection. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub and primary interaction surface of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four splayed legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom face features four square sockets to receive the leg tenons. + +### 2~5. Four Legs (Leg 01, Leg 02, Leg 03, Leg 04) +The supporting entities of the stool. +* **Component Purpose**: Vertical and lateral support. Transfers the seat load to the ground, utilizing a splayed angle to ensure anti-overturning stability. +* **Assembly Direction**: Inserted upwards along the splayed axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). The top features a tenon that interference-fits into the seat panel. The mid-section features mortises to receive the stretcher tenons. + +### 6~7. Stretchers (Stretcher 01, Stretcher 02) +The cross-bracing structure of the stool. +* **Component Purpose**: Connects the legs diagonally in an X-pattern to prevent splaying under load and drastically increase the overall structural rigidity of the frame. +* **Assembly Direction**: Inserted horizontally/diagonally between opposite pairs of legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mid-section mortises of the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 7-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **Stretcher 01 -> Leg 01 & Leg 04** | Joint: interlocking | Note: Stretcher ends inserted into the mid-section mortises of opposite legs. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Stretcher ends inserted into the mid-section mortises of the other pair of opposite legs. +* **Seat Panel -> All Legs** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_tapered_legs/harness.ts b/eval/tasks/muse-stool_tapered_legs/harness.ts new file mode 100644 index 000000000..190adaada --- /dev/null +++ b/eval/tasks/muse-stool_tapered_legs/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_tapered_legs/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_tapered_legs' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_tapered_legs'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_tapered_legs/prompt.md b/eval/tasks/muse-stool_tapered_legs/prompt.md new file mode 100644 index 000000000..cdddf481f --- /dev/null +++ b/eval/tasks/muse-stool_tapered_legs/prompt.md @@ -0,0 +1,113 @@ +# stool_tapered_legs (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a rounded-square stool with tapered legs and reinforced braces designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 330.0 mm × 330.0 mm × 452.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four tapered legs; six reinforced braces (stretchers). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +11 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Leg 04 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 +- Stretcher 05 +- Stretcher 06 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Ensures adequate structural strength for seating while accommodating the insertion depth of the leg tenons. +- **leg_height**: 434.0 (314.0 ~ 594.0 mm). Determines the seating height, strictly following ergonomic standards for single-person seating posture. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Determines the bite depth of the physical connections into the seat panel. +- **seat_width**: 330.0 (240.0 ~ 420.0 mm). Defines the primary seating area width. +- **seat_depth**: 330.0 (240.0 ~ 420.0 mm). Defines the primary seating area depth. +- **leg_top_size**: 20.0 (18.0 ~ 30.0 mm). Defines the cross-section size of the leg at the top, affecting the joint strength at the seat interface. +- **leg_bottom_size**: 36.0 (28.0 ~ 48.0 mm). Defines the base footprint of the leg, ensuring anti-overturning stability. +- **tenon_size**: 8.0 (6.0 ~ 12.0 mm). Controls the thickness of the tenon for the leg-to-seat joint. +- **stretcher_z**: 154.0 (104.0 ~ 234.0 mm). Sets the vertical position of the primary perimeter stretchers for leg reinforcement. +- **stretcher_secondary_z**: 212.0 (152.0 ~ 292.0 mm). Sets the vertical position of the secondary stretchers (cross braces) to prevent interference with the primary stretchers. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Determines the vertical stiffness and load-bearing capacity of the stretcher bars. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the horizontal stiffness of the stretcher bars and the thickness of their tenons. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features four sockets generated via boolean cut to receive the leg tenons. + +### 2~5. Four Legs (Leg 01, Leg 02, Leg 03, Leg 04) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Features mortises along the shaft to receive the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a tenon of height `tenon_height` that interference-fits into the bottom sockets of the seat panel. The sides feature mortises to connect with the stretchers. + +### 6~11. Six Stretchers (Stretcher 01 to Stretcher 06) +The horizontal reinforcement entities of the stool. +* **Component Purpose**: Horizontal reinforcement. Connects the legs to prevent splaying under load and increases overall structural rigidity. Configured in a "box_x" mode (perimeter box + internal cross). +* **Assembly Direction**: Inserted horizontally between the respective legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 11-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 1. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 2. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 3. +* **Leg 04 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's socket 4. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Perimeter brace connecting adjacent legs. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Perimeter brace connecting adjacent legs. +* **Stretcher 03 -> Leg 03 & Leg 04** | Joint: interlocking | Note: Perimeter brace connecting adjacent legs. +* **Stretcher 04 -> Leg 04 & Leg 01** | Joint: interlocking | Note: Perimeter brace connecting adjacent legs. +* **Stretcher 05 -> Leg 01 & Leg 03** | Joint: interlocking | Note: Diagonal cross brace connecting opposite legs. +* **Stretcher 06 -> Leg 02 & Leg 04** | Joint: interlocking | Note: Diagonal cross brace connecting opposite legs. +* **Seat Panel -> All Legs** | Joint: Support Base | Note: Acts as the core hub; all connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_tripod_bar/harness.ts b/eval/tasks/muse-stool_tripod_bar/harness.ts new file mode 100644 index 000000000..14b680987 --- /dev/null +++ b/eval/tasks/muse-stool_tripod_bar/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_tripod_bar/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_tripod_bar' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_tripod_bar'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_tripod_bar/prompt.md b/eval/tasks/muse-stool_tripod_bar/prompt.md new file mode 100644 index 000000000..f0722b7d3 --- /dev/null +++ b/eval/tasks/muse-stool_tripod_bar/prompt.md @@ -0,0 +1,116 @@ +# stool_tripod_bar (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a tall tripod bar stool with two triangular stretcher levels designed for wood-based assembly. + +## Geometry and Dimensions +Approx. 312.0 mm × 312.0 mm × 778.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating at an elevated bar or counter. + +## Structural Features +Seat panel; three legs; six stretchers (forming two triangular bracing levels). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +10 + +## Component Names +- Seat panel +- Leg 01 +- Leg 02 +- Leg 03 +- Stretcher 01 +- Stretcher 02 +- Stretcher 03 +- Stretcher 04 +- Stretcher 05 +- Stretcher 06 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Must be thick enough to accommodate the insertion depth of the leg tenons and support the user's weight. +- **leg_height**: 760.0 (640.0 ~ 920.0 mm). Determines the overall seating height, strictly following ergonomic standards for bar stools. +- **tenon_height**: 9.0 (6.0 ~ 13.0 mm). Determines the bite depth of the physical connections between the legs and the seat panel. +- **seat_radius**: 156.0 (120.0 ~ 201.0 mm). Defines the seating area to ensure ergonomic comfort and prevent tipping. +- **leg_top_radius**: 11.0 (8.0 ~ 16.0 mm). Ensures sufficient material at the top of the leg to support the tenon and bear the seat load. +- **leg_bottom_radius**: 14.0 (10.0 ~ 20.0 mm). Provides a wider base for the leg to ensure anti-overturning stability on the ground. +- **tenon_radius**: 5.0 (4.0 ~ 8.0 mm). Controls the thickness of the cylindrical tenon to prevent shear failure. +- **stretcher_z**: 312.0 (262.0 ~ 392.0 mm). Sets the height of the lower stretcher level, acting as a structural brace and potential footrest. +- **stretcher_secondary_z**: 644.0 (584.0 ~ 724.0 mm). Sets the height of the upper stretcher level for additional torsional rigidity. +- **stretcher_bar_width**: 14.0 (10.0 ~ 20.0 mm). Determines the horizontal stiffness of the bracing components. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Determines the vertical load-bearing capacity of the stretchers. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub of the stool. +* **Component Purpose**: Acts as the main load-bearing base for seating and provides localization references and mechanical interfaces (sockets) for the three legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). Bottom features three cylindrical sockets arranged radially. + +### 2~4. Legs (Leg 01, Leg 02, Leg 03) +The supporting entities of the stool. +* **Component Purpose**: Vertical support. Transfers the seat load to the ground, ensuring anti-overturning stability. Features mortises along its length to receive the stretchers. +* **Assembly Direction**: Inserted upwards along the +Z axis into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Top features a cylindrical tenon that interference-fits into the bottom sockets of the seat panel. The body features mortise cuts to receive stretcher tenons. + +### 5~7. Lower Stretchers (Stretcher 01, Stretcher 02, Stretcher 03) +The primary horizontal bracing entities. +* **Component Purpose**: Connects the legs at the lower level (`stretcher_z`) to prevent splaying and increase the overall structural rigidity of the tripod base. +* **Assembly Direction**: Inserted horizontally between adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +### 8~10. Upper Stretchers (Stretcher 04, Stretcher 05, Stretcher 06) +The secondary horizontal bracing entities. +* **Component Purpose**: Connects the legs at the upper level (`stretcher_secondary_z`) to provide additional resistance against torsional forces and reinforce the upper leg structure. +* **Assembly Direction**: Inserted horizontally between adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 10-component model: + +* **Leg 01 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's first radial socket. +* **Leg 02 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's second radial socket. +* **Leg 03 -> Seat Panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's third radial socket. +* **Stretcher 01 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Lower stretcher connecting Leg 01 and Leg 02. +* **Stretcher 02 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Lower stretcher connecting Leg 02 and Leg 03. +* **Stretcher 03 -> Leg 03 & Leg 01** | Joint: interlocking | Note: Lower stretcher connecting Leg 03 and Leg 01. +* **Stretcher 04 -> Leg 01 & Leg 02** | Joint: interlocking | Note: Upper stretcher connecting Leg 01 and Leg 02. +* **Stretcher 05 -> Leg 02 & Leg 03** | Joint: interlocking | Note: Upper stretcher connecting Leg 02 and Leg 03. +* **Stretcher 06 -> Leg 03 & Leg 01** | Joint: interlocking | Note: Upper stretcher connecting Leg 03 and Leg 01. +* **Seat Panel -> All Leg Components** | Joint: Support Base | Note: Acts as the core hub; all leg connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-stool_wide_splayed/harness.ts b/eval/tasks/muse-stool_wide_splayed/harness.ts new file mode 100644 index 000000000..38e2e7e32 --- /dev/null +++ b/eval/tasks/muse-stool_wide_splayed/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-stool_wide_splayed/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'stool_wide_splayed' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-stool_wide_splayed'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-stool_wide_splayed/prompt.md b/eval/tasks/muse-stool_wide_splayed/prompt.md new file mode 100644 index 000000000..ffa0aa9d6 --- /dev/null +++ b/eval/tasks/muse-stool_wide_splayed/prompt.md @@ -0,0 +1,111 @@ +# stool_wide_splayed (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a wide splayed stool featuring a rectangular seat and two levels of perimeter rails (stretchers) for enhanced structural stability. + +## Geometry and Dimensions +Approx. 360.0 mm × 360.0 mm × 448.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +interlocking + +## Mechanical Condition +Single-person seating. + +## Structural Features +Seat panel; four splayed legs; eight perimeter stretchers (arranged in two levels). + +## Special Requirements +Keep assembly split unchanged. + +## Planned Component Quantity +13 + +## Component Names +- seat_panel +- leg_01 +- leg_02 +- leg_03 +- leg_04 +- stretcher_01 +- stretcher_02 +- stretcher_03 +- stretcher_04 +- stretcher_05 +- stretcher_06 +- stretcher_07 +- stretcher_08 + +## Adjustable Parameters +- **seat_thickness**: 18.0 (10.0 ~ 32.0 mm). Ensures sufficient material thickness to support seating loads and accommodate leg tenons without breaking. +- **leg_height**: 430.0 (310.0 ~ 590.0 mm). Determines the primary seating height, conforming to ergonomic standards for stools. +- **tenon_height**: 8.0 (5.0 ~ 12.0 mm). Controls the insertion depth of the leg tenons into the seat panel sockets. +- **seat_width**: 360.0 (270.0 ~ 450.0 mm). Defines the lateral seating area. +- **seat_depth**: 360.0 (270.0 ~ 450.0 mm). Defines the longitudinal seating area. +- **leg_top_size**: 26.0 (18.0 ~ 36.0 mm). Sets the cross-sectional size of the leg at the top to ensure adequate joint strength with the seat. +- **leg_bottom_size**: 34.0 (26.0 ~ 46.0 mm). Sets the cross-sectional size of the leg at the floor level for a stable footprint. +- **tenon_size**: 9.5 (6.5 ~ 13.5 mm). Determines the thickness of the tenon, balancing joint strength and remaining leg material. +- **stretcher_z**: 148.0 (98.0 ~ 228.0 mm). Sets the vertical position of the first (lower) level of perimeter stretchers to prevent leg splay. +- **stretcher_secondary_z**: 214.0 (154.0 ~ 294.0 mm). Sets the vertical position of the second (upper) level of perimeter stretchers for additional torsional rigidity. +- **stretcher_bar_width**: 15.0 (11.0 ~ 21.0 mm). Defines the vertical width of the stretcher bars for bending resistance. +- **stretcher_bar_thickness**: 10.0 (8.0 ~ 14.0 mm). Defines the horizontal thickness of the stretcher bars. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1. Seat Panel +The central hub and primary contact surface of the stool. +* **Component Purpose**: Acts as the main load-bearing base for the user and provides localization references and mechanical interfaces (sockets) for the four legs. +* **Assembly Direction**: Fixed base component, positioned at absolute $Z = leg\_height$. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted; potential micro-sliding if loose). The bottom face features four sockets to receive the leg tenons. + +### 2~5. Four Legs (leg_01 to leg_04) +The supporting entities of the stool. +* **Component Purpose**: Vertical and splayed support. Transfers the seat load to the ground while providing mortises to receive the stretcher network. +* **Assembly Direction**: Inserted upwards along the +Z axis (with splay angles) into the seat panel. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). The top features a tenon inserted into the seat panel. The sides feature mortises to receive the stretcher tenons. + +### 6~13. Eight Stretchers (stretcher_01 to stretcher_08) +The horizontal bracing entities of the stool. +* **Component Purpose**: Connects the legs at two different height levels to form a rigid perimeter frame, preventing the legs from splaying outward under load and increasing overall structural stiffness. +* **Assembly Direction**: Inserted horizontally/diagonally between adjacent legs. +* **Connection & Kinematics**: interlocking (Rigid when interference-fitted). Both ends of each stretcher feature tenons that insert into the corresponding mortises on the legs. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 13-component model: + +* **leg_01 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_02 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_03 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **leg_04 -> seat_panel** | Joint: interlocking | Note: Leg top tenon inserted into seat's corresponding socket. +* **stretcher_01 ~ stretcher_04 -> leg_01 ~ leg_04** | Joint: interlocking | Note: Lower level stretchers connecting adjacent legs via end tenons. +* **stretcher_05 ~ stretcher_08 -> leg_01 ~ leg_04** | Joint: interlocking | Note: Upper level stretchers connecting adjacent legs via end tenons. +* **seat_panel -> All Components** | Joint: Support Base | Note: Acts as the core hub; leg connection sockets generated via boolean cut. diff --git a/eval/tasks/muse-toothbrush_holder/harness.ts b/eval/tasks/muse-toothbrush_holder/harness.ts new file mode 100644 index 000000000..5c159e44b --- /dev/null +++ b/eval/tasks/muse-toothbrush_holder/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-toothbrush_holder/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'toothbrush_holder' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-toothbrush_holder'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-toothbrush_holder/prompt.md b/eval/tasks/muse-toothbrush_holder/prompt.md new file mode 100644 index 000000000..3536bf5bb --- /dev/null +++ b/eval/tasks/muse-toothbrush_holder/prompt.md @@ -0,0 +1,73 @@ +# toothbrush_holder (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a parametric toothbrush holder with a hollow internal cavity and integrated drainage cutouts for moisture-resistant bathroom storage. + +## Geometry and Dimensions +Approx. 200.0 mm × 100.0 mm × 80.0 mm. + +## Material +ABS + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static load-bearing storage for lightweight items (toothbrushes, toothpaste) in a moisture-rich environment. + +## Structural Features +Hollow curved main body; two cylindrical drainage holes; two spherical relief cutouts. + +## Special Requirements +Ensure the boolean difference cutters (cylinders and spheres) fully intersect the inner hollow cavity to guarantee unobstructed water drainage. + +## Planned Component Quantity +1 + +## Component Names +- toothbrush_holder_body + +## Adjustable Parameters +- **body_length**: 200 (100.0 ~ 280.0 mm). Controls the overall width and storage capacity of the holder. +- **body_height**: 80 (40.0 ~ 140.0 mm). Determines the vertical depth of the storage slot to prevent tall items from tipping over. +- **body_depth**: 100 (50.0 ~ 180.0 mm). Controls the front-to-back footprint and extrusion depth of the holder. +- **wall_offset**: 6 (3.0 ~ 16.0 mm). Defines the wall thickness to ensure structural rigidity during FDM printing. +- **hole_radius**: 15 (8.0 ~ 28.0 mm). Sizes the cylindrical drainage holes to allow water escape without letting toothbrushes slip through. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main hollow profile using a boundary surface between the inner rectangular slot and the outer bezier curve chain. +2. Extrude the surface along the Y-axis to form the main body. +3. Execute boolean difference operations using positioned cylinders and spheres to create the drainage and relief features. + +--- + +### 1. toothbrush_holder_body +The primary and sole structural entity of the model. +* **Component Purpose**: Acts as the main storage receptacle, providing physical containment for toothbrushes while allowing water to drain through the bottom cutouts. +* **Assembly Direction**: Not applicable (Standalone component). +* **Connection & Kinematics**: Not applicable (Single solid body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **toothbrush_holder_body -> World/Ground** | Joint: Support Base | Note: Standalone single-piece design; no internal assembly joints. diff --git a/eval/tasks/muse-vase/harness.ts b/eval/tasks/muse-vase/harness.ts new file mode 100644 index 000000000..748e40da6 --- /dev/null +++ b/eval/tasks/muse-vase/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase/prompt.md b/eval/tasks/muse-vase/prompt.md new file mode 100644 index 000000000..ad5e8ca71 --- /dev/null +++ b/eval/tasks/muse-vase/prompt.md @@ -0,0 +1,76 @@ +# vase (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a parametric, thin-walled vase with a continuous sine-wave modulated profile, designed to serve as a decorative container. + +## Geometry and Dimensions +Approx. 75.0 mm × 75.0 mm × 177.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static display and containment; resting on a flat surface to hold lightweight decorative items, dried flowers, or act as a standalone aesthetic piece. + +## Structural Features +Wavy lofted outer shell; hollow interior cavity; solid bottom base (2mm thick); flat top rim. + +## Special Requirements +The final geometry must be sewn into a single, watertight closed solid (TopAbs_SOLID) to ensure printability and structural integrity of the thin walls. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **vase_height**: 177 (80.0 ~ 320.0 mm). Determines the overall vertical capacity and aesthetic proportion of the vase. +- **profile_radius_scale**: 25 (10.0 ~ 60.0 mm). Controls the base scaling factor for the vase's radius, directly affecting the overall volume and footprint. +- **wall_thickness**: 2 (1.0 ~ 8.0 mm). Defines the shell thickness; the lower limit ensures FDM printability without gaps, while the upper limit provides structural rigidity. +- **profile_segments**: 10 (4 ~ 24). Determines the vertical resolution and the number of control wire profiles used for the lofting operation. +- **profile_wave_cycles**: 1 (0.5 ~ 3.0). Controls the number of sine wave undulations (bulges and constrictions) along the height of the vase. +- **profile_phase_span**: 1 (0.5 ~ 2.0). Defines the span of the sine wave phase evaluated from the bottom to the top profile. +- **profile_phase_start**: 0 (-0.5 ~ 0.5). Sets the initial phase of the sine wave at the base, determining whether the base starts at a bulge or a constriction. +- **profile_radius_offset**: 0.5 (0.3 ~ 1.2). Provides a baseline offset to the radius calculation to ensure the inner diameter remains positive and the vase maintains a minimum functional width. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate a series of outer and inner circular wires along the Z-axis based on sine-wave modulated radii. +2. Lift the first inner wire by the specified wall thickness to create a solid floor. +3. Loft the outer wires and inner wires separately to create the main shell surfaces. +4. Generate the bottom cap and top ring surfaces. +5. Sew all surfaces (inner floor, inner loft, top ring, outer loft, bottom cap) into a single closed solid. + +--- + +### 1. vase_body +The primary and only structural body of the vase. +* **Component Purpose**: Acts as the main container and aesthetic exterior, providing both the internal cavity for holding items and the stable base for resting on flat surfaces. +* **Assembly Direction**: Not applicable (Standalone component). +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single continuous solid model; no assembly required. diff --git a/eval/tasks/muse-vase_amphora_soft/harness.ts b/eval/tasks/muse-vase_amphora_soft/harness.ts new file mode 100644 index 000000000..ac3e1ffc8 --- /dev/null +++ b/eval/tasks/muse-vase_amphora_soft/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_amphora_soft/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_amphora_soft' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_amphora_soft'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_amphora_soft/prompt.md b/eval/tasks/muse-vase_amphora_soft/prompt.md new file mode 100644 index 000000000..1d38d10b4 --- /dev/null +++ b/eval/tasks/muse-vase_amphora_soft/prompt.md @@ -0,0 +1,59 @@ +# vase_amphora_soft (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a soft amphora vase with a lifted shoulder and a slightly flared lip, intended for use as a decorative vessel. + +## Geometry and Dimensions +Approx. 80.0 mm × 80.0 mm × 234.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static decorative display; capable of holding lightweight items such as dried flowers. + +## Structural Features +Continuous lofted outer shell; hollow interior cavity; solid bottom base; flared top lip. + +## Special Requirements +The final geometry must be successfully sewed into a closed, watertight solid. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 234.0 (174.0 ~ 314.0 mm). Determines the overall vertical extent of the vase. +- **wall_thickness**: 2.5 (1.5 ~ 4.5 mm). Controls the thickness of the vase shell, ensuring structural integrity and printability. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution and the number of cross-sectional wires used for lofting the surface. +- **profile_radius**: 40.0 (10.0 ~ 56.0 mm). Controls the radial bounds of the amphora profile (derived from the maximum value in the profile points). + +## Component Details + +### 1. vase_body +The primary and sole component of the model, forming the complete amphora shape. +* **Component Purpose**: Acts as a decorative vessel and containment shell. +* **Assembly Direction**: N/A (Base standalone component, built vertically along the +Z axis). +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single continuous solid body; no assembly required. diff --git a/eval/tasks/muse-vase_bell_short/harness.ts b/eval/tasks/muse-vase_bell_short/harness.ts new file mode 100644 index 000000000..cb64cbdd3 --- /dev/null +++ b/eval/tasks/muse-vase_bell_short/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_bell_short/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_bell_short' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_bell_short'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_bell_short/prompt.md b/eval/tasks/muse-vase_bell_short/prompt.md new file mode 100644 index 000000000..83dd2817f --- /dev/null +++ b/eval/tasks/muse-vase_bell_short/prompt.md @@ -0,0 +1,72 @@ +# vase_bell_short (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a short bell-shaped vase with a generous lower chamber and a gentle taper, designed for aesthetic display and containment. + +## Geometry and Dimensions +Approx. 104.0 mm × 104.0 mm × 168.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static aesthetic display, holding lightweight items (e.g., dried flowers) without internal pressure. + +## Structural Features +Single hollow vase body; closed bottom floor; open top ring; lofted smooth or wavy wall profile. + +## Special Requirements +Maintain uniform wall thickness throughout the lofted profile to ensure successful slicing and structural integrity during 3D printing. + +## Planned Component Quantity +1 + +## Component Names +- Vase body + +## Adjustable Parameters +- **height**: 168.0 (120.0 ~ 248.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 2.8 (1.6 ~ 4.8 mm). Determines the shell thickness, balancing material usage, print time, and structural rigidity. +- **steps**: 12 (10.0 ~ 22.0). Defines the vertical resolution and the number of lofting sections used to generate the smooth or wavy profile. +- **profile_radius**: 52.0 (14.0 ~ 68.0 mm). Controls the radial bounds of the spline profile to shape the bell curve, with the maximum radius dictating the overall width. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate the base profile points and interpolate the radii along the Z-axis. +2. Create inner and outer wire cross-sections based on the wall thickness. +3. Loft the wires to create the inner and outer shells, and cap the top and bottom to sew into a single solid. + +--- + +### 1. Vase body +The primary and sole component of the model. +* **Component Purpose**: Acts as the main container, providing the internal volume and external aesthetic shape. +* **Assembly Direction**: Not applicable (Base component, built vertically along the +Z axis). +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **Vase body -> Standalone** | Joint: None | Note: Single-component design, no assembly required. diff --git a/eval/tasks/muse-vase_bottle_round/harness.ts b/eval/tasks/muse-vase_bottle_round/harness.ts new file mode 100644 index 000000000..110ceb593 --- /dev/null +++ b/eval/tasks/muse-vase_bottle_round/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_bottle_round/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_bottle_round' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_bottle_round'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_bottle_round/prompt.md b/eval/tasks/muse-vase_bottle_round/prompt.md new file mode 100644 index 000000000..b7e6f3dce --- /dev/null +++ b/eval/tasks/muse-vase_bottle_round/prompt.md @@ -0,0 +1,72 @@ +# vase_bottle_round (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a bottle-style vase with a round belly and a long slender neck, designed as a freestanding decorative container. + +## Geometry and Dimensions +Approx. 88.0 mm × 88.0 mm × 248.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Freestanding decorative object, capable of holding lightweight items (e.g., dried or artificial flowers) and supporting its own weight. + +## Structural Features +Continuous outer shell; hollow interior cavity; closed bottom base; open top neck. + +## Special Requirements +The model must remain a single, continuous, closed solid shell to ensure proper slicing and structural integrity during the 3D printing process. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 248.0 (188.0 ~ 328.0 mm). Controls the overall vertical extent of the vase. +- **wall_thickness**: 2.4 (1.5 ~ 4.4 mm). Determines the thickness of the vase shell, ensuring structural stability and printability without excessive material use. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution and the number of lofting sections used to generate the smooth or wavy profile. +- **profile_radius**: 44.0 (10.0 ~ 60.0 mm). Controls the radial extent of the belly and neck at various height fractions to shape the vase's silhouette. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate the outer and inner wire profiles based on the interpolated radius points and wall thickness. +2. Loft the wires to create the inner and outer shells. +3. Cap the bottom and connect the top ring to form a single sewed solid body. + +--- + +### 1. vase_body +The main and only structural body of the vase. +* **Component Purpose**: Acts as the decorative outer shell and internal container for holding items. +* **Assembly Direction**: N/A (Manufactured in place vertically along the +Z axis from the base). +* **Connection & Kinematics**: N/A (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **vase_body -> Standalone** | Joint: N/A | Note: Single continuous body, no assembly required. diff --git a/eval/tasks/muse-vase_bowl_low/harness.ts b/eval/tasks/muse-vase_bowl_low/harness.ts new file mode 100644 index 000000000..5a52076da --- /dev/null +++ b/eval/tasks/muse-vase_bowl_low/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_bowl_low/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_bowl_low' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_bowl_low'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_bowl_low/prompt.md b/eval/tasks/muse-vase_bowl_low/prompt.md new file mode 100644 index 000000000..d2966b7d3 --- /dev/null +++ b/eval/tasks/muse-vase_bowl_low/prompt.md @@ -0,0 +1,73 @@ +# vase_bowl_low (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a low flower bowl vase with a wide mouth and shallow body, featuring a parametric profile capable of generating smooth or wavy aesthetic variations. + +## Geometry and Dimensions +Approx. 116.0 mm × 116.0 mm × 142.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static decorative display, suitable for holding lightweight items, dried flowers, or acting as a standalone aesthetic centerpiece. + +## Structural Features +Single continuous shell; solid bottom base; hollow interior cavity; lofted outer and inner walls; top connecting ring. + +## Special Requirements +The final sewed shape must remain a closed, watertight solid with consistent wall thickness to ensure printability without internal voids. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 142.0 (120.0 ~ 222.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Ensures structural integrity during FDM printing while maximizing internal volume; lower limit prevents fragile walls. +- **steps**: 12 (10.0 ~ 22.0). Determines the vertical resolution and number of cross-sectional layers used to loft the geometry. +- **profile_radius**: 58.0 (10.0 ~ 74.0 mm). Controls the radial extent of the vase profile at various heights to define the bowl's curvature. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate horizontal circular or wavy wire profiles for both the inner and outer boundaries based on the height steps. +2. Loft the outer wires to create the exterior surface and the inner wires to create the interior cavity. +3. Cap the bottom and bridge the top gap with a ruled surface. +4. Sew all surfaces together into a single valid solid body. + +--- + +### 1. vase_body +The main and only entity of the model. +* **Component Purpose**: Acts as the decorative vessel, providing internal volume for contents while maintaining structural stability on a flat surface. +* **Assembly Direction**: Not applicable (Base component, built vertically along the +Z axis). +* **Connection & Kinematics**: Not applicable (Single solid body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **vase_body -> Standalone** | Joint: None | Note: Single continuous body; no assembly required. diff --git a/eval/tasks/muse-vase_bud_slim/harness.ts b/eval/tasks/muse-vase_bud_slim/harness.ts new file mode 100644 index 000000000..bb5d8eb86 --- /dev/null +++ b/eval/tasks/muse-vase_bud_slim/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_bud_slim/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_bud_slim' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_bud_slim'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_bud_slim/prompt.md b/eval/tasks/muse-vase_bud_slim/prompt.md new file mode 100644 index 000000000..2e6cbfe76 --- /dev/null +++ b/eval/tasks/muse-vase_bud_slim/prompt.md @@ -0,0 +1,59 @@ +# vase_bud_slim (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a slim bud vase with a narrow mouth and a softly swollen body, intended for aesthetic display and holding small floral arrangements. + +## Geometry and Dimensions +Approx. 76.0 mm × 76.0 mm × 208.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static tabletop display; acts as a lightweight container for single buds or small dried flowers. + +## Structural Features +Single hollow vase body; solid bottom base; narrow top opening; lofted curved shell. + +## Special Requirements +The final geometry must be sewn into a completely closed, watertight solid shell to ensure structural integrity and proper slicing for 3D printing. + +## Planned Component Quantity +1 + +## Component Names +- Vase body + +## Adjustable Parameters +- **height**: 208.0 (148.0 ~ 288.0 mm). Determines the overall vertical extent of the vase. +- **wall_thickness**: 2.4 (1.5 ~ 4.4 mm). Controls the shell thickness; the lower limit ensures printability and structural stability, while the upper limit prevents excessive material use. +- **steps**: 14 (10.0 ~ 24.0). Defines the vertical resolution and the number of lofting sections used to interpolate the smooth profile. +- **profile_radius**: Variable (10.0 ~ 54.0 mm). Controls the radial swelling and narrowing of the vase body at various height fractions (defined by `profile_points`). + +## Component Details + +### 1. Vase body +The primary and sole component of the model. +* **Component Purpose**: Acts as the functional container and provides the aesthetic outer profile. Formed by lofting outer and inner wire profiles and sewing them with a bottom cap and top ring. +* **Assembly Direction**: N/A (Manufactured as a single piece, built vertically along the +Z axis). +* **Connection & Kinematics**: None (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +Vase body -> Standalone | Joint: None | Note: Single-piece construction; no assembly required. diff --git a/eval/tasks/muse-vase_column_neck/harness.ts b/eval/tasks/muse-vase_column_neck/harness.ts new file mode 100644 index 000000000..a481743cf --- /dev/null +++ b/eval/tasks/muse-vase_column_neck/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_column_neck/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_column_neck' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_column_neck'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_column_neck/prompt.md b/eval/tasks/muse-vase_column_neck/prompt.md new file mode 100644 index 000000000..272d7aeb9 --- /dev/null +++ b/eval/tasks/muse-vase_column_neck/prompt.md @@ -0,0 +1,73 @@ +# vase_column_neck (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Column vase with straight upper walls and a stable rounded base, designed as a decorative container. + +## Geometry and Dimensions +Approx. 72.0 mm × 72.0 mm × 258.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Decorative container standing vertically on a flat surface, capable of holding lightweight or dry items. + +## Structural Features +Hollow cylindrical body; stable rounded base; straight upper walls; uniform wall thickness; smooth or wave-textured lofted exterior. + +## Special Requirements +Ensure the bottom cap, inner floor, lofted walls, and top ring are perfectly sewed to form a closed, watertight solid shell. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 258.0 (198.0 ~ 338.0 mm). Determines the overall vertical extent of the vase. +- **wall_thickness**: 2.7 (1.5 ~ 4.7 mm). Defines the structural thickness of the vase walls, balancing material usage, print time, and rigidity. +- **steps**: 18 (12.0 ~ 28.0). Controls the vertical resolution and number of cross-sections used for lofting the profile. +- **profile_radius**: Variable (10.0 ~ 52.0 mm). Controls the radial extent of the vase profile at various height intervals to shape the rounded base and straight neck. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate the outer and inner wire profiles based on the interpolated radius points and wall thickness. +2. Loft the cross-sectional wires to create the inner and outer shell surfaces. +3. Cap the top and bottom with ruled surfaces and planar faces. +4. Sew all surfaces together into a single valid solid body. + +--- + +### 1. vase_body +The main structural and aesthetic entity of the vase. +* **Component Purpose**: Acts as the primary container, providing the exterior aesthetic profile and the hollow interior volume. +* **Assembly Direction**: N/A (Standalone component). +* **Connection & Kinematics**: N/A (Single continuous solid body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **vase_body -> Standalone** | Joint: None | Note: Single continuous solid body; no assembly required. diff --git a/eval/tasks/muse-vase_gourd_tall/harness.ts b/eval/tasks/muse-vase_gourd_tall/harness.ts new file mode 100644 index 000000000..6d0bec34e --- /dev/null +++ b/eval/tasks/muse-vase_gourd_tall/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_gourd_tall/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_gourd_tall' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_gourd_tall'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_gourd_tall/prompt.md b/eval/tasks/muse-vase_gourd_tall/prompt.md new file mode 100644 index 000000000..a7f05900b --- /dev/null +++ b/eval/tasks/muse-vase_gourd_tall/prompt.md @@ -0,0 +1,59 @@ +# vase_gourd_tall (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a tall gourd vase with two rounded lobes and a restrained neck for aesthetic display and floral arrangements. + +## Geometry and Dimensions +Approx. 84.0 mm × 84.0 mm × 236.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Freestanding desktop decor, non-load-bearing aesthetic model. + +## Structural Features +Closed bottom base; thin-walled double-lobed body; open top neck. + +## Special Requirements +The exported STEP must remain a closed solid shell with uniform wall thickness to ensure printability. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 236.0 (176.0 ~ 316.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 2.6 (1.5 ~ 4.6 mm). Determines the shell thickness, balancing material usage, print time, and structural rigidity. +- **steps**: 16 (10.0 ~ 26.0). Defines the vertical resolution and the number of lofting sections used to generate the smooth or wavy profile. +- **profile_radius**: 42.0 (10.0 ~ 58.0 mm). Controls the maximum radial extent of the gourd lobes to define the volumetric capacity and footprint. + +## Component Details + +### 1. vase_body +The primary and sole component forming the gourd vase. +* **Component Purpose**: Acts as the decorative container, providing the external aesthetic gourd shape and internal hollow volume. +* **Assembly Direction**: Placed vertically along the +Z axis (freestanding base). +* **Connection & Kinematics**: Not applicable (Single solid body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Ground | Joint: Support Base | Note: Standalone object; no internal assembly required. diff --git a/eval/tasks/muse-vase_lantern_soft/harness.ts b/eval/tasks/muse-vase_lantern_soft/harness.ts new file mode 100644 index 000000000..59ff1d224 --- /dev/null +++ b/eval/tasks/muse-vase_lantern_soft/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_lantern_soft/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_lantern_soft' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_lantern_soft'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_lantern_soft/prompt.md b/eval/tasks/muse-vase_lantern_soft/prompt.md new file mode 100644 index 000000000..ef4425e4c --- /dev/null +++ b/eval/tasks/muse-vase_lantern_soft/prompt.md @@ -0,0 +1,71 @@ +# vase_lantern_soft (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a lantern-shaped decorative vase with a rounded midsection, short neck, and a hollow internal cavity for floral display. + +## Geometry and Dimensions +Approx. 100.0 mm × 100.0 mm × 196.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static decorative display, resting vertically on a flat surface. + +## Structural Features +Hollow lantern-shaped body; closed bottom base; open top neck; continuous thin-wall shell. + +## Special Requirements +The model must remain a closed, manifold solid (watertight) to ensure proper slicing and potential liquid containment. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 196.0 (136.0 ~ 276.0 mm). Controls the overall vertical extent of the vase. +- **wall_thickness**: 2.6 (1.5 ~ 4.6 mm). Determines the shell thickness, ensuring adequate structural integrity and printability without excessive material use. +- **steps**: 14 (10.0 ~ 24.0). Defines the vertical resolution and the number of lofting sections used to generate the smooth curved profile. +- **profile_radius**: 50.0 (10.0 ~ 66.0 mm). Controls the radial bounds of the vase's midsection and neck, dictating the volumetric capacity and lantern-like curvature. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate the outer and inner circular wire profiles based on the interpolated radius function along the Z-axis. +2. Loft the outer wires to create the exterior surface and the inner wires to create the interior cavity. +3. Cap the bottom with planar faces and connect the top edges using a ruled surface. +4. Sew all surfaces together into a single watertight solid. + +--- + +### 1. vase_body +The primary and only structural entity of the model. +* **Component Purpose**: Acts as the main decorative shell and containment vessel. +* **Assembly Direction**: Standalone component, built vertically along the +Z axis. +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single-piece monolithic design without physical assembly interfaces. diff --git a/eval/tasks/muse-vase_urn_classic/harness.ts b/eval/tasks/muse-vase_urn_classic/harness.ts new file mode 100644 index 000000000..7b8142e62 --- /dev/null +++ b/eval/tasks/muse-vase_urn_classic/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_urn_classic/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_urn_classic' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_urn_classic'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_urn_classic/prompt.md b/eval/tasks/muse-vase_urn_classic/prompt.md new file mode 100644 index 000000000..a6f0587b1 --- /dev/null +++ b/eval/tasks/muse-vase_urn_classic/prompt.md @@ -0,0 +1,59 @@ +# vase_urn_classic (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a classic urn vase with a broad shoulder and an upright neck, featuring a smooth, continuous lofted shell and a hollow interior for aesthetic display. + +## Geometry and Dimensions +Approx. 92.0 mm × 92.0 mm × 226.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable +## Mechanical Condition +Static aesthetic display, freestanding on a flat surface. + +## Structural Features +Continuous lofted outer shell; hollow internal cavity; flat bottom base; upright neck. + +## Special Requirements +The exported STEP must remain a closed, watertight solid shell to ensure proper slicing and 3D printability. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 226.0 (166.0 ~ 306.0 mm). Determines the overall vertical extent of the vase. +- **wall_thickness**: 2.8 (1.6 ~ 4.8 mm). Ensures structural integrity and printability of the hollow shell without excessive material use. +- **steps**: 16 (10.0 ~ 26.0). Controls the vertical resolution and discretization of the lofted profile layers. +- **profile_radius**: Variable (11.0 ~ 62.0 mm). Constrains the radial bounds of the spline profile to maintain the classic urn proportion. + +## Component Details + +### 1. vase_body +The primary and sole continuous entity of the model. +* **Component Purpose**: Acts as the main structural and aesthetic body, providing an internal cavity defined by the offset wall thickness. +* **Assembly Direction**: Not applicable (Standalone base component). +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single continuous body requiring no assembly. diff --git a/eval/tasks/muse-vase_wave_blossom/harness.ts b/eval/tasks/muse-vase_wave_blossom/harness.ts new file mode 100644 index 000000000..5f5444cd4 --- /dev/null +++ b/eval/tasks/muse-vase_wave_blossom/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_blossom/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_blossom' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_blossom'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_blossom/prompt.md b/eval/tasks/muse-vase_wave_blossom/prompt.md new file mode 100644 index 000000000..ea5fb5fd8 --- /dev/null +++ b/eval/tasks/muse-vase_wave_blossom/prompt.md @@ -0,0 +1,75 @@ +# vase_wave_blossom (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a decorative blossom vase with a fuller upper body and a gently ruffled opening, featuring a continuous, parametrically generated wavy shell. + +## Geometry and Dimensions +Approx. 85.0 mm × 85.0 mm × 202.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +N/A (Single solid body) + +## Mechanical Condition +Static decorative display; suitable for holding lightweight dried flowers or acting as a standalone aesthetic centerpiece. + +## Structural Features +Wavy outer shell; matching inner shell offset by wall thickness; solid bottom base; ruffled top rim connecting the inner and outer shells. + +## Special Requirements +Must maintain a continuous, non-intersecting manifold shell to ensure successful slicing and 3D printing. Overhangs must be kept gradual to avoid the need for internal support structures. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 202.0 (142.0 ~ 282.0 mm). Controls the total vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Determines the shell thickness, ensuring structural integrity and printability without excessive material use. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution (number of layers) used to loft the vase profile. +- **pts_per_layer**: 120 (96.0 ~ 156.0). Defines the horizontal resolution of the spline curves for each layer. +- **wave_count**: 6 (4.0 ~ 10.0). Determines the number of primary petals or ruffles distributed around the circumference. +- **twist**: 0.018 (0.0 ~ 0.048). Applies a helical twist to the wave pattern along the Z-axis, creating a dynamic sweeping effect. +- **secondary_amp**: 0.8 (0.0 ~ 2.0). Controls the amplitude of secondary, higher-frequency ripples superimposed on the primary waves for added surface texture. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate horizontal spline profiles for the inner and outer shells based on base radius, wave amplitude, and twist parameters. +2. Loft the outer profiles to create the exterior surface and the inner profiles to create the interior cavity. +3. Cap the bottom with planar faces and connect the top inner and outer wires with a ruled surface. +4. Sew all surfaces together into a single, watertight solid body. + +--- + +### 1. vase_body +The sole structural and aesthetic entity of the design. +* **Component Purpose**: Acts as the primary decorative vessel, containing the internal volume while displaying the complex parametric wave pattern on the exterior. +* **Assembly Direction**: N/A (Standalone part, printed vertically from the bottom base upwards along the +Z axis). +* **Connection & Kinematics**: N/A (Single solid body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single continuous body; no assembly required. diff --git a/eval/tasks/muse-vase_wave_dune/harness.ts b/eval/tasks/muse-vase_wave_dune/harness.ts new file mode 100644 index 000000000..3c414cf93 --- /dev/null +++ b/eval/tasks/muse-vase_wave_dune/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_dune/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_dune' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_dune'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_dune/prompt.md b/eval/tasks/muse-vase_wave_dune/prompt.md new file mode 100644 index 000000000..985a6ce9b --- /dev/null +++ b/eval/tasks/muse-vase_wave_dune/prompt.md @@ -0,0 +1,77 @@ +# vase_wave_dune (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a dune-like decorative vase featuring a broad body with slow rolling, mathematically driven wave surface patterns. + +## Geometry and Dimensions +Approx. 105.0 mm × 105.0 mm × 188.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +Not applicable (Single solid body) + +## Mechanical Condition +Freestanding decorative object, suitable for standalone display or holding lightweight items (e.g., dried flowers). + +## Structural Features +Single hollow body; closed bottom base; open top rim; undulating lofted outer wall. + +## Special Requirements +The generated geometry must maintain a continuous, manifold shell to ensure successful slicing and 3D printing. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 188.0 (128.0 ~ 268.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.2 (2.0 ~ 5.2 mm). Determines the shell thickness for structural integrity and printability. +- **steps**: 16 (10.0 ~ 26.0). Defines the vertical resolution (number of layers) for the lofting operation. +- **pts_per_layer**: 96 (72.0 ~ 132.0). Defines the radial resolution (number of points) for the spline curves on each layer. +- **wave_count**: 4 (3.0 ~ 8.0). Number of primary wave undulations around the circumference. +- **twist**: 0.012 (0.0 ~ 0.042). Controls the helical twist of the waves along the Z-axis. +- **secondary_amp**: 0.4 (0.0 ~ 1.6). Amplitude of the secondary high-frequency waves for surface texture. +- **profile_radius**: 48.0 (14.0 ~ 64.0 mm). Base radius range for the vase profile. +- **wave_amp**: 4.6 (0.8 ~ 6.6 mm). Amplitude of the primary waves. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate inner and outer spline profiles layer by layer based on mathematical wave functions. +2. Loft the profiles to create the inner and outer shells. +3. Cap the top and bottom with ruled surfaces and faces. +4. Sew all surfaces into a single solid body. + +--- + +### 1. vase_body +The main and only structural entity of the vase. +* **Component Purpose**: Acts as the decorative shell and container. +* **Assembly Direction**: Built vertically from the base (+Z axis) during 3D printing. +* **Connection & Kinematics**: Not applicable (Single solid body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single continuous part. diff --git a/eval/tasks/muse-vase_wave_fluted/harness.ts b/eval/tasks/muse-vase_wave_fluted/harness.ts new file mode 100644 index 000000000..61aaf48d0 --- /dev/null +++ b/eval/tasks/muse-vase_wave_fluted/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_fluted/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_fluted' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_fluted'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_fluted/prompt.md b/eval/tasks/muse-vase_wave_fluted/prompt.md new file mode 100644 index 000000000..d58900f86 --- /dev/null +++ b/eval/tasks/muse-vase_wave_fluted/prompt.md @@ -0,0 +1,79 @@ +# vase_wave_fluted (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative fluted wave vase with rhythmic vertical undulations, designed as a single continuous shell for aesthetic display. + +## Geometry and Dimensions +Approx. 91.0 mm × 91.0 mm × 214.0 mm. + +## Material +PLA + +## Manufacturing Method +FDM 3D Printing + +## Connection Method (Joint Type) +Not applicable (Single monolithic component) + +## Mechanical Condition +Freestanding decorative container, suitable for holding lightweight items such as dried flowers or acting as a standalone aesthetic piece. + +## Structural Features +Hollow lofted shell; solid bottom base; fluted exterior and interior walls with primary and secondary wave patterns. + +## Special Requirements +The lofted surfaces must be perfectly sewn to form a closed, watertight solid. Overhangs must be kept within printable limits to avoid the need for internal supports during FDM printing. + +## Planned Component Quantity +1 + +## Component Names +- Vase body + +## Adjustable Parameters +- **height**: 214.0 (154.0 ~ 294.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Determines the structural thickness of the shell; lower limits ensure printability, while upper limits prevent excessive material use. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution (number of layers) used to generate the lofted shape. +- **pts_per_layer**: 108 (84.0 ~ 144.0). Defines the horizontal resolution of the spline curves for smooth wave generation. +- **wave_count**: 8 (6.0 ~ 12.0). Sets the number of primary vertical flutes around the circumference. +- **twist**: 0.01 (0.0 ~ 0.04). Controls the helical rotation of the flutes along the Z-axis. +- **secondary_amp**: 0.8 (0.0 ~ 2.0 mm). Controls the intensity of the secondary surface ripples for added texture. +- **profile_radius**: 40.0 (12.0 ~ 56.0 mm). Base radius constraint for the profile points, determining the overall width of the vase. +- **wave_amp**: 5.4 (1.0 ~ 7.4 mm). Controls the depth of the primary flutes. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate horizontal spline wires for the inner and outer profiles based on the wave and radius parameters. +2. Loft the outer wires and inner wires independently. +3. Cap the top and bottom with ruled surfaces and planar faces. +4. Sew all surfaces together to form a valid solid. + +--- + +### 1. Vase body +The primary and sole entity of the model. +* **Component Purpose**: Acts as the main aesthetic body and functional container. +* **Assembly Direction**: Freestanding base component, built vertically along the +Z axis. +* **Connection & Kinematics**: Not applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **Vase body -> Ground** | Joint: None | Note: Standalone monolithic object. diff --git a/eval/tasks/muse-vase_wave_petal/harness.ts b/eval/tasks/muse-vase_wave_petal/harness.ts new file mode 100644 index 000000000..6db810e05 --- /dev/null +++ b/eval/tasks/muse-vase_wave_petal/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_petal/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_petal' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_petal'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_petal/prompt.md b/eval/tasks/muse-vase_wave_petal/prompt.md new file mode 100644 index 000000000..8887f41d4 --- /dev/null +++ b/eval/tasks/muse-vase_wave_petal/prompt.md @@ -0,0 +1,77 @@ +# vase_wave_petal (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative petal-mouth vase featuring a wavy, twisted outer surface and a hollow interior, designed with a calm lower body and stronger top ripples. + +## Geometry and Dimensions +Approx. 90.0 mm × 90.0 mm × 214.0 mm. + +## Material +PLA + +## Manufacturing Method +FDM 3D Printing + +## Connection Method (Joint Type) +N/A (Single monolithic body) + +## Mechanical Condition +Freestanding decorative object, suitable for holding lightweight items (e.g., dried flowers) or serving as a standalone aesthetic model. + +## Structural Features +Wavy outer shell; hollow interior cavity; solid bottom base; petal-shaped top opening. + +## Special Requirements +The lofted inner and outer shells must be perfectly sewed into a closed, manifold solid to ensure printability without slicing errors. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 214.0 (154.0 ~ 294.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Ensures sufficient thickness for 3D printing perimeters and overall structural stability. +- **steps**: 18 (12.0 ~ 28.0). Determines the vertical resolution (number of layers) used to generate the lofted profiles. +- **pts_per_layer**: 120 (96.0 ~ 156.0). Defines the horizontal resolution of the spline curves to ensure smooth wave transitions. +- **wave_count**: 7 (5.0 ~ 11.0). Sets the number of primary petal folds (waves) around the vase circumference. +- **twist**: 0.015 (0.0 ~ 0.045). Controls the helical rotation of the wave pattern along the Z-axis from bottom to top. +- **secondary_amp**: 0.5 (0.0 ~ 1.7 mm). Defines the intensity of the secondary high-frequency surface ripples. +- **profile_radius**: (11.0 ~ 56.0 mm). Constrains the base radius scaling of the vase's vertical profile. +- **wave_amp**: (0.0 ~ 8.8 mm). Controls the maximum amplitude of the wave deformations at the petal mouth. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate the base profile points and wave amplitude functions. +2. Create inner and outer spline wires for each vertical step, applying twist and wave transformations. +3. Loft the outer wires and inner wires separately. +4. Cap the top and bottom with ruled surfaces and faces, then sew all surfaces into a single solid. + +--- + +### 1. vase_body +The primary and sole structural entity of the model. +* **Component Purpose**: Acts as the decorative container, providing both the aesthetic exterior and the functional hollow interior. +* **Assembly Direction**: N/A (Freestanding base component, built vertically along the +Z axis). +* **Connection & Kinematics**: N/A (Single solid body). + +--- + +## Component Assembly Graph (Textual) +* **vase_body -> Standalone** | Joint: N/A | Note: Single continuous solid body; no assembly required. diff --git a/eval/tasks/muse-vase_wave_ripple/harness.ts b/eval/tasks/muse-vase_wave_ripple/harness.ts new file mode 100644 index 000000000..a352e03b1 --- /dev/null +++ b/eval/tasks/muse-vase_wave_ripple/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_ripple/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_ripple' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_ripple'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_ripple/prompt.md b/eval/tasks/muse-vase_wave_ripple/prompt.md new file mode 100644 index 000000000..d7960798c --- /dev/null +++ b/eval/tasks/muse-vase_wave_ripple/prompt.md @@ -0,0 +1,65 @@ +# vase_wave_ripple (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative ripple vase with fine lobes, a straightened upper neck, and a hollow interior designed for aesthetic display. + +## Geometry and Dimensions +Approx. 82.4 mm × 82.4 mm × 226.0 mm. + +## Material +PLA + +## Manufacturing Method +FDM 3D Printing + +## Connection Method (Joint Type) +N/A (Single continuous body) + +## Mechanical Condition +Static decorative display; capable of holding lightweight items such as dried flowers. + +## Structural Features +Hollow vase body; rippled/lobed outer wall; flat base; open top neck. + +## Special Requirements +The lofted shell must remain a closed, manifold solid to ensure proper slicing and 3D printability. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 226.0 (166.0 ~ 306.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 2.8 (1.6 ~ 4.8 mm). Determines the shell thickness, ensuring printability and structural integrity. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution (number of lofting layers) for the B-Rep generation. +- **pts_per_layer**: 132 (108.0 ~ 168.0). Defines the radial resolution for the spline curves. +- **wave_count**: 12 (10.0 ~ 16.0). Sets the number of primary lobes/ripples around the vase circumference. +- **twist**: 0.02 (0.0 ~ 0.05). Controls the helical twist of the ripples along the Z-axis. +- **secondary_amp**: 0.6 (0.0 ~ 1.8). Sets the amplitude of the secondary (finer) ripples for surface texture. +- **profile_radius**: (10.0 ~ 54.0 mm). Constrains the base radius of the vase profile. +- **wave_amp**: (0.2 ~ 5.2 mm). Controls the amplitude of the primary waves. + +## Component Details + +### 1. vase_body +The main and only component of the vase, featuring a complex organic exterior and a hollowed interior. +* **Component Purpose**: Acts as a decorative container and aesthetic display piece. +* **Assembly Direction**: N/A (Printed in place, built vertically along the +Z axis). +* **Connection & Kinematics**: N/A (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: N/A | Note: Single continuous solid body; no assembly required. diff --git a/eval/tasks/muse-vase_wave_scallop/harness.ts b/eval/tasks/muse-vase_wave_scallop/harness.ts new file mode 100644 index 000000000..ba2ee465f --- /dev/null +++ b/eval/tasks/muse-vase_wave_scallop/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_scallop/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_scallop' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_scallop'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_scallop/prompt.md b/eval/tasks/muse-vase_wave_scallop/prompt.md new file mode 100644 index 000000000..7dde37ac8 --- /dev/null +++ b/eval/tasks/muse-vase_wave_scallop/prompt.md @@ -0,0 +1,76 @@ +# vase_wave_scallop (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative scalloped vase with broad lobes, a small pinched opening, and a wave-like organic surface for aesthetic display. + +## Geometry and Dimensions +Approx. 90.0 mm × 90.0 mm × 206.0 mm. + +## Material +PLA + +## Manufacturing Method +FDM 3D Printing + +## Connection Method (Joint Type) +Not applicable (Single-piece component) + +## Mechanical Condition +Static decorative display, freestanding container. + +## Structural Features +Single continuous hollow shell; scalloped outer lobes; pinched top opening; flat bottom base. + +## Special Requirements +Maintain smooth lofting between spline layers to prevent non-manifold geometry; ensure continuous outer perimeters for optimal 3D printing without supports. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 206.0 (146.0 ~ 286.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Determines the shell thickness for structural rigidity and printability. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution (number of lofting layers) for the smooth surface generation. +- **pts_per_layer**: 108 (84.0 ~ 144.0). Controls the horizontal resolution of the spline curves forming the scalloped waves. +- **wave_count**: 5 (3.0 ~ 9.0). Sets the number of primary lobes/scallops around the circumference. +- **twist**: 0.008 (0.0 ~ 0.038). Determines the helical rotation of the wave pattern along the Z-axis. +- **secondary_amp**: 0.7 (0.0 ~ 1.9 mm). Adds secondary high-frequency ripples to the primary wave profile for complex texturing. +- **profile_radius**: (10.0 ~ 54.0 mm). Controls the base radial profile of the vase at various heights. +- **wave_amp**: (0.2 ~ 8.2 mm). Controls the depth/amplitude of the primary scalloped lobes. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate inner and outer spline wires based on radial and wave amplitude functions. +2. Loft the wires vertically to create the inner and outer shells. +3. Cap the top and bottom with ruled surfaces/faces and sew into a single watertight solid. + +--- + +### 1. vase_body +The main and only structural body of the vase. +* **Component Purpose**: Acts as a decorative container and aesthetic display piece. +* **Assembly Direction**: Not applicable (freestanding base positioned at absolute Z = 0). +* **Connection & Kinematics**: Not applicable (Single-piece component). + +--- + +## Component Assembly Graph (Textual) +* **vase_body -> Standalone** | Joint: None | Note: Single continuous part; no assembly required. diff --git a/eval/tasks/muse-vase_wave_shell/harness.ts b/eval/tasks/muse-vase_wave_shell/harness.ts new file mode 100644 index 000000000..bef1972b9 --- /dev/null +++ b/eval/tasks/muse-vase_wave_shell/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_shell/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_shell' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_shell'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_shell/prompt.md b/eval/tasks/muse-vase_wave_shell/prompt.md new file mode 100644 index 000000000..f63c3f167 --- /dev/null +++ b/eval/tasks/muse-vase_wave_shell/prompt.md @@ -0,0 +1,77 @@ +# vase_wave_shell (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative, shell-like vase with a wavy, twisted outer surface and a hollow interior, designed primarily for aesthetic display. + +## Geometry and Dimensions +Approx. 95.0 mm × 95.0 mm × 212.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +N/A (Single solid body) + +## Mechanical Condition +Freestanding decorative object; suitable for holding lightweight dried flowers or acting as a standalone aesthetic centerpiece. + +## Structural Features +Continuous wavy outer shell; hollow interior cavity; flat base; undulating top lip. + +## Special Requirements +Must maintain a continuous, watertight shell (manifold solid) to ensure proper slicing and toolpath generation for 3D printing. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 212.0 (152.0 ~ 292.0 mm). Controls the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Determines the shell thickness, ensuring structural integrity and printability without requiring internal supports. +- **steps**: 18 (12.0 ~ 28.0). Defines the vertical resolution of the lofted layers. +- **pts_per_layer**: 108 (84.0 ~ 144.0). Defines the radial resolution of the spline points per layer. +- **wave_count**: 9 (7.0 ~ 13.0). Number of primary undulations/waves distributed around the circumference. +- **twist**: 0.028 (0.008 ~ 0.058). Controls the helical twist of the waves along the Z-axis. +- **secondary_amp**: 1.0 (0.2 ~ 2.2 mm). Amplitude of the secondary wave, adding complex surface texture to the primary undulations. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate radial spline profiles for both the inner and outer shells at varying Z-heights based on the wave and twist parameters. +2. Loft the outer wires to create the exterior surface and the inner wires to create the interior cavity. +3. Cap the bottom and bridge the top lip using ruled surfaces. +4. Sew all surfaces together into a single, closed solid body. + +--- + +### 1. vase_body +The primary and only component of the model. +* **Component Purpose**: Acts as a decorative vessel, providing both the external aesthetic form and the internal containment volume. +* **Assembly Direction**: N/A (Freestanding base component, built vertically along the +Z axis). +* **Connection & Kinematics**: N/A (Single solid body). + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 1-component model: + +* **vase_body -> Standalone** | Joint: None | Note: Single continuous solid body; no assembly required. diff --git a/eval/tasks/muse-vase_wave_twist/harness.ts b/eval/tasks/muse-vase_wave_twist/harness.ts new file mode 100644 index 000000000..c096e62ab --- /dev/null +++ b/eval/tasks/muse-vase_wave_twist/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-vase_wave_twist/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'vase_wave_twist' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-vase_wave_twist'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-vase_wave_twist/prompt.md b/eval/tasks/muse-vase_wave_twist/prompt.md new file mode 100644 index 000000000..3ab7bc1b9 --- /dev/null +++ b/eval/tasks/muse-vase_wave_twist/prompt.md @@ -0,0 +1,76 @@ +# vase_wave_twist (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create a decorative vase featuring a twisted wave profile with a moderate spiral motion along its body, designed as a single continuous shell for additive manufacturing. + +## Geometry and Dimensions +Approx. 101.0 mm × 101.0 mm × 230.0 mm. + +## Material +PLA + +## Manufacturing Method +FDM 3D Printing + +## Connection Method (Joint Type) +Not Applicable (Single solid body) + +## Mechanical Condition +Static decorative display, suitable for holding lightweight dried flowers or acting as a standalone aesthetic centerpiece. + +## Structural Features +Hollow twisted body; solid bottom base; open top rim; undulating inner and outer walls. + +## Special Requirements +The lofted shell must remain a closed, manifold solid to ensure successful slicing and 3D printing. The inner wall must strictly offset from the outer wall to maintain a consistent wall thickness. + +## Planned Component Quantity +1 + +## Component Names +- vase_body + +## Adjustable Parameters +- **height**: 230.0 (170.0 ~ 310.0 mm). Determines the overall vertical dimension of the vase. +- **wall_thickness**: 3.0 (1.8 ~ 5.0 mm). Ensures sufficient shell thickness for FDM printability and structural stability. +- **steps**: 20 (14.0 ~ 30.0). Defines the number of vertical layers used to construct the lofted surface. +- **pts_per_layer**: 120 (96.0 ~ 156.0). Defines the radial resolution of the spline curves for each layer. +- **wave_count**: 6 (4.0 ~ 10.0). Sets the number of primary ridges/waves around the circumference of the vase. +- **twist**: 0.045 (0.025 ~ 0.075). Controls the degree of spiral torsion applied along the Z-axis. +- **secondary_amp**: 1.0 (0.2 ~ 2.2 mm). Amplitude of the secondary high-frequency wave, adding surface texture. +- **profile_radius**: (10.0 ~ 60.0 mm). Controls the base radius of the vase profile at different heights. +- **wave_amp**: (0.8 ~ 7.6 mm). Controls the amplitude of the primary wave deformation. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Generate horizontal spline profiles for the inner and outer walls at varying Z-heights based on wave and twist functions. +2. Loft the outer wires to form the exterior surface and the inner wires to form the interior cavity. +3. Cap the bottom and sew the top ring to form a single, watertight solid body. + +--- + +### 1. vase_body +The primary and sole structural entity of the design. +* **Component Purpose**: Acts as the main decorative shell and container. +* **Assembly Direction**: Base component, built vertically along the +Z axis. +* **Connection & Kinematics**: Not Applicable (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +vase_body -> Standalone | Joint: None | Note: Single solid body generated via lofting; no assembly required. diff --git a/eval/tasks/muse-wave_vase/harness.ts b/eval/tasks/muse-wave_vase/harness.ts new file mode 100644 index 000000000..7d09bae83 --- /dev/null +++ b/eval/tasks/muse-wave_vase/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-wave_vase/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'wave_vase' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-wave_vase'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-wave_vase/prompt.md b/eval/tasks/muse-wave_vase/prompt.md new file mode 100644 index 000000000..76f6e9b71 --- /dev/null +++ b/eval/tasks/muse-wave_vase/prompt.md @@ -0,0 +1,65 @@ +# wave_vase (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Create an aesthetic, parametric vase featuring a twisted, corrugated outer profile and a hollow interior, primarily intended for decorative desktop display. + +## Geometry and Dimensions +Approx. 190.0 mm × 190.0 mm × 250.0 mm. + +## Material +PLA + +## Manufacturing Method +3D Printing + +## Connection Method (Joint Type) +N/A (Single solid body) + +## Mechanical Condition +Desktop decoration, non-load-bearing aesthetic display, or holding lightweight dried flowers. + +## Structural Features +Twisted wavy outer shell; hollow interior cavity; solid flat base; top rim. + +## Special Requirements +The final geometry must be sewed and exported as a single closed manifold solid to ensure proper slicing for 3D printing. + +## Planned Component Quantity +1 + +## Component Names +- wave_vase_body + +## Adjustable Parameters +- **base_radius**: 50 (20.0 ~ 120.0 mm). Determines the base footprint and the starting width of the vase. +- **height**: 250 (120.0 ~ 420.0 mm). Controls the overall vertical dimension of the vase. +- **profile_amp**: 40 (0.0 ~ 80.0 mm). Controls the outward bulge (amplitude) of the vase's overall silhouette. +- **wave_amp**: 5 (0.0 ~ 20.0 mm). Defines the depth and prominence of the surface ripples/corrugations. +- **thickness**: 4 (1.0 ~ 10.0 mm). Sets the wall thickness between the inner and outer lofted shells to ensure printability and structural integrity. +- **steps**: 20 (8 ~ 40). Resolution parameter defining the number of vertical layers used for lofting the shape. +- **pts_per_layer**: 100 (48 ~ 180). Resolution parameter defining the number of points used to construct the horizontal splines. +- **twist**: 0.05 (0.0 ~ 0.12). Defines the helical rotation rate of the waves along the Z-axis, creating the twisting effect. +- **wave_count**: 8 (3 ~ 16). Sets the number of wave ridges distributed around the perimeter of the vase. + +## Component Details + +### 1. wave_vase_body +The main and only component of the model. +* **Component Purpose**: Acts as a decorative container. +* **Assembly Direction**: N/A (Standalone base component). +* **Connection & Kinematics**: N/A (Single continuous body). + +--- + +## Component Assembly Graph (Textual) +wave_vase_body -> Standalone | Joint: None | Note: Single continuous body constructed from sewed inner/outer lofts and end faces. diff --git a/eval/tasks/muse-workbench/harness.ts b/eval/tasks/muse-workbench/harness.ts new file mode 100644 index 000000000..dfd6bd53f --- /dev/null +++ b/eval/tasks/muse-workbench/harness.ts @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +// eval/tasks/muse-workbench/harness.ts +// +// Auto-generated by eval/lib/importMuseTasks.ts from the MUSE benchmark +// case 'workbench' (dataset: huggingface.co/datasets/dongxiaoyu/MUSE). +// +// Gates (kernelCAD product gates, always on): +// - evaluates clean (no blocking diagnostics) +// +// Scored fields (MUSE's own funnel stages, run by MUSE's code via +// eval/oracle/museScorer.ts — no thresholds of ours): +// - 'muse stage1: sandbox executes' +// - 'muse stage2: component overlap-free' +// +// The OCCT-validity stage (watertight / manifold / self-intersection) +// requires MUSE's external 'validator' module, which is not published; +// its availability and raw output are surfaced via metrics. + +import { evaluateScript } from '../../oracle/kernelcad-client'; +import { runMuseScorer } from '../../oracle/museScorer'; +import type { HarnessCtx, HarnessResult } from '../../types'; + +export default async function harness(scriptPath: string, ctx?: HarnessCtx): Promise { + const ev = await evaluateScript(scriptPath); + if (!ev.ok) { + return { gates: { 'evaluates clean': false }, scored: {} }; + } + + let sandboxOk = false; + let overlapFree = false; + let metrics: Record | undefined; + if (ctx) { + try { + const result = await runMuseScorer(scriptPath, ctx.runDir, 'muse-workbench'); + sandboxOk = result.sandboxOk; + // Per MUSE funnel semantics a stage not reached counts as failed. + overlapFree = result.overlapFree === true; + metrics = { + muse_reason: result.reason, + muse_sandbox_ok: result.sandboxOk, + muse_sandbox_error: result.sandboxError.slice(0, 500), + muse_step_exists: result.stepExists, + muse_solid_count: result.resultSolidCount, + muse_bbox: result.bbox.join(','), + muse_validator_available: result.validatorAvailable, + muse_geometry_valid: result.geometry ? (result.geometry.geometry_valid as boolean) : null, + muse_geometry_note: result.geometryNote, + muse_overlap_free: result.overlapFree, + muse_max_overlap_ratio: result.interpenetration?.max_overlap_ratio ?? null, + muse_interpenetrating_pairs: result.interpenetration?.interpenetrating_pairs ?? null, + muse_render_ok: result.renderOk, + muse_render_png: result.renderPngPath, + muse_render_step: result.renderStepPath, + muse_export_ms: result.exportMs, + muse_score_ms: result.scoreMs, + muse_errors: result.errors.join(' | ').slice(0, 1000) || null, + }; + } catch (err) { + metrics = { + muse_sandbox_ok: false, + muse_reason: `scorer threw: ${err instanceof Error ? err.message : String(err)}`, + }; + } + } + + return { + gates: { 'evaluates clean': true }, + scored: { + 'muse stage1: sandbox executes': sandboxOk, + 'muse stage2: component overlap-free': overlapFree, + }, + ...(metrics !== undefined ? { metrics } : {}), + }; +} diff --git a/eval/tasks/muse-workbench/prompt.md b/eval/tasks/muse-workbench/prompt.md new file mode 100644 index 000000000..fa90b8cd2 --- /dev/null +++ b/eval/tasks/muse-workbench/prompt.md @@ -0,0 +1,140 @@ +# workbench (imported from the MUSE text-to-CAD benchmark) + +Build the following design as a kernelCAD `.kcad.ts` script. Millimetres, Z-up, degrees. + +If the specification calls for multiple independent components, return a +multi-part model — one solid per named component — and preserve the +component count, proportions, and assembly intent from the specification. + +## Design Specification + +# Design Specification + +## Design Goal +Construct a sturdy, slatted wooden workbench with a lower storage shelf, designed for CNC-machined timber assembly. + +## Geometry and Dimensions +Approx. 1200.0 mm × 600.0 mm × 864.0 mm. + +## Material +Timber + +## Manufacturing Method +CNC Milling + +## Connection Method (Joint Type) +Nailing + +## Mechanical Condition +Load-bearing workspace for manual tasks and lower shelf for tool/material storage. + +## Structural Features +Slatted top surface; four vertical legs; under-top side rails; bottom side rails; bottom cross rails; slatted lower shelf; back brace rail. + +## Special Requirements +Keep assembly split unchanged. Ensure all cylindrical pilot holes align perfectly for dowel/fastener insertion across intersecting components. + +## Planned Component Quantity +39 + +## Component Names +- top_slat_01 ~ top_slat_13 +- left_under_top_rail +- right_under_top_rail +- right_front_leg +- right_back_leg +- left_front_leg +- left_back_leg +- right_bottom_rail +- left_bottom_rail +- back_bottom_cross +- front_bottom_cross +- left_shelf_rail +- right_shelf_rail +- shelf_slat_01 ~ shelf_slat_13 +- back_brace_rail + +## Adjustable Parameters +- **bench_width**: 1200.0 (800.0 ~ 1600.0 mm). Defines the overall span of the workspace. +- **bench_depth**: 600.0 (400.0 ~ 800.0 mm). Defines the working depth and determines the total number of slats required. +- **bench_height**: 850.0 (750.0 ~ 950.0 mm). Ergonomic height for standing or seated work. +- **shelf_height**: 200.0 (100.0 ~ 400.0 mm). Determines the vertical clearance and position of the lower storage shelf. +- **board_thickness**: 14.0 (10.0 ~ 22.0 mm). Ensures adequate structural rigidity for the boards and slats. +- **board_width**: 40.0 (30.0 ~ 60.0 mm). Defines the width of the structural framing and individual slats. +- **slat_gap**: 5.0 (2.0 ~ 10.0 mm). Controls the spacing between top and shelf slats for drainage, expansion, or tool clearance. +- **hole_radius**: 1.5 (0.5 ~ 3.0 mm). Sizes the pilot holes for connecting dowels or standard fasteners. +- **hole_depth**: 4.0 (1.0 ~ 8.0 mm). Determines the insertion depth for the connecting dowels/fasteners into the boards. + +## Component Details + +**Global Output Requirements** +1. The component must remain an independent geometric body. +2. The exported STEP must remain a closed solid. + +**Global Modeling Steps** +1. Build the main profile of the part based on the original script. +2. Complete key features like holes, slots, lofts, or chamfers. +3. Place the part back in its original position within the sample assembly. + +--- + +### 1~13. Top Slats +The primary working surface of the bench. +* **Component Purpose**: Provides a flat, slatted load-bearing area for work activities. +* **Assembly Direction**: Placed downwards along the -Z axis onto the under-top rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Bottom faces feature blind holes that align with the under-top rails. + +### 14~15. Under-top Rails (Left, Right) +The upper longitudinal supports. +* **Component Purpose**: Supports the top slats and ties the front and back legs together at the top of the structure. +* **Assembly Direction**: Horizontal insertion along the Y axis between the legs. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features Z-direction holes for the top slats and Y-direction holes for the legs. + +### 16~19. Four Legs (Right Front, Right Back, Left Front, Left Back) +The main vertical support entities. +* **Component Purpose**: Transfers all loads to the ground and provides mounting points for all horizontal rails. +* **Assembly Direction**: Vertical standing along the +Z axis. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features multiple Y-direction and X-direction holes at varying heights to accept side rails, cross rails, and braces. + +### 20~21. Bottom Side Rails (Right, Left) +The lower longitudinal supports. +* **Component Purpose**: Connects the front and back legs near the floor to prevent longitudinal racking. +* **Assembly Direction**: Horizontal insertion along the Y axis between the legs. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features Y-direction holes at the ends. + +### 22~23. Bottom Cross Rails (Back, Front) +The lower transverse supports. +* **Component Purpose**: Connects the left and right legs near the floor to prevent lateral racking. +* **Assembly Direction**: Horizontal insertion along the X axis between the legs. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features X-direction holes at the ends. + +### 24~25. Shelf Side Rails (Left, Right) +The mid-level longitudinal supports. +* **Component Purpose**: Provides the structural base for the lower shelf slats. +* **Assembly Direction**: Horizontal insertion along the Y axis between the legs. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features Z-direction holes for the shelf slats and Y-direction holes for the legs. + +### 26~38. Shelf Slats +The secondary storage surface. +* **Component Purpose**: Provides a slatted platform for storing tools and materials beneath the main workspace. +* **Assembly Direction**: Placed downwards along the -Z axis onto the shelf side rails. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Bottom faces feature blind holes that align with the shelf rails. + +### 39. Back Brace Rail +The mid-level transverse support. +* **Component Purpose**: Connects the rear legs at mid-height to provide additional lateral stiffness and prevent wobbling. +* **Assembly Direction**: Horizontal insertion along the X axis between the back legs. +* **Connection & Kinematics**: Dowel Joint (Constrains 2 translations + 2 rotations). Features X-direction holes at the ends. + +--- + +## Component Assembly Graph (Textual) +Based on the logical mapping of the 39-component model: + +* **Top Slats -> Under-top Rails** | Joint: Dowel Joint | Note: Slats rest on rails with Z-direction alignment holes. +* **Under-top Rails -> Legs** | Joint: Dowel Joint | Note: Rails connect to inner faces of legs via Y-direction holes. +* **Bottom Side Rails -> Legs** | Joint: Dowel Joint | Note: Rails connect to inner faces of legs near the base via Y-direction holes. +* **Shelf Side Rails -> Legs** | Joint: Dowel Joint | Note: Rails connect to inner faces of legs at shelf height via Y-direction holes. +* **Shelf Slats -> Shelf Side Rails** | Joint: Dowel Joint | Note: Slats rest on shelf rails with Z-direction alignment holes. +* **Bottom Cross Rails -> Legs** | Joint: Dowel Joint | Note: Cross rails connect to outer faces of legs via X-direction holes. +* **Back Brace Rail -> Back Legs** | Joint: Dowel Joint | Note: Brace connects to back legs at mid-height via X-direction holes. diff --git a/scripts/museJudge.test.ts b/scripts/museJudge.test.ts new file mode 100644 index 000000000..59e8a20b3 --- /dev/null +++ b/scripts/museJudge.test.ts @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { EventEmitter } from 'node:events'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { judgeCase, mapJudgePayload } from './museJudge'; + +function fakeSpawn(payload: Record, outPath: string) { + return ((_cmd: string, _args: string[]) => { + const child = new EventEmitter() as unknown as { + stdout: EventEmitter; + stderr: EventEmitter; + on: EventEmitter['on']; + emit: EventEmitter['emit']; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + setImmediate(() => { + writeFileSync(outPath, JSON.stringify(payload, null, 2)); + child.stdout.emit('data', `${JSON.stringify(payload)}\n`); + child.emit('close', 0); + }); + return child; + }) as unknown as typeof import('node:child_process').spawn; +} + +describe('mapJudgePayload', () => { + it('maps six categories and computes overall fallback', () => { + const out = mapJudgePayload({ + overall_score_normalized: 0.5, + items: [ + { category_en: 'Assembly Readiness', score: 1 }, + { category_en: 'Joint Design', score: 0 }, + { category_en: 'Tolerance', score: 1 }, + { category_en: 'Functional Adaptation', score: 1 }, + { category_en: 'Usage Stability', score: 0 }, + { category_en: 'Manufacturability', score: 1 }, + ], + }); + expect(out.categories.assembly_readiness).toBe(1); + expect(out.categories.joint_design).toBe(0); + expect(out.overall).toBe(0.5); + }); + + it('writes a forced-zero judge.json without spawning the wrapper', async () => { + const dir = mkdtempSync(join(tmpdir(), 'judge-')); + const spawn = vi.fn(); + const result = await judgeCase( + { + caseName: 'stool', + datasetCaseDir: '/nonexistent', + candidatePng: '/nonexistent/render.png', + outPath: join(dir, 'judge.json'), + museRoot: '/nonexistent', + pythonBin: 'python3', + baseUrl: 'https://example.invalid', + model: 'judge', + }, + false, + spawn as unknown as typeof import('node:child_process').spawn, + ); + expect(result.forcedZero).toBe(true); + expect(result.overall).toBe(0); + expect(spawn).not.toHaveBeenCalled(); + }); + + it('maps a real wrapper payload from the spawn stdout into judge.json', async () => { + const dir = mkdtempSync(join(tmpdir(), 'judge-')); + const outPath = join(dir, 'judge.json'); + const payload = { + overall: 0.5, + categories: {}, + summary: 'half', + items: [ + { category_en: 'Assembly Readiness', score: 1 }, + { category_en: 'Joint Design', score: 0 }, + { category_en: 'Tolerance', score: 1 }, + { category_en: 'Functional Adaptation', score: 1 }, + { category_en: 'Usage Stability', score: 0 }, + { category_en: 'Manufacturability', score: 1 }, + ], + }; + const result = await judgeCase( + { + caseName: 'stool', + datasetCaseDir: '/nonexistent', + candidatePng: '/nonexistent/render.png', + outPath, + museRoot: '/nonexistent', + pythonBin: 'python3', + baseUrl: 'https://example.invalid', + model: 'judge', + }, + true, + fakeSpawn(payload, outPath), + ); + expect(result.forcedZero).toBe(false); + expect(result.overall).toBe(0.5); + expect(result.categories.assembly_readiness).toBe(1); + expect(result.categories.joint_design).toBe(0); + const written = JSON.parse( + (await import('node:fs')).readFileSync(outPath, 'utf8'), + ) as { raw?: { items?: unknown[] } }; + expect(written.raw?.items).toHaveLength(6); + }); +}); diff --git a/scripts/museJudge.ts b/scripts/museJudge.ts new file mode 100644 index 000000000..1560a0fa0 --- /dev/null +++ b/scripts/museJudge.ts @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { spawn as nodeSpawn } from 'node:child_process'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ZERO_CATEGORIES, type JudgeCategories } from '../eval/lib/museAggregate'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const WRAPPER_PY = resolve(__dirname, '../eval/oracle/museJudgeWrapper.py'); + +/** Outer cap on the python judge process (the HTTP call has its own 180s timeout). */ +const JUDGE_SPAWN_TIMEOUT_MS = Number(process.env.MUSE_JUDGE_TIMEOUT_MS ?? 240_000); + +export interface JudgeCaseArgs { + caseName: string; + datasetCaseDir: string; + candidatePng: string; + outPath: string; + museRoot: string; + pythonBin: string; + baseUrl: string; + model: string; + timeoutMs?: number; +} + +export interface JudgeResult { + overall: number; + categories: JudgeCategories; + forcedZero: boolean; + forcedZeroReason?: string; + summary?: string; +} + +type SpawnFn = typeof nodeSpawn; + +const CATEGORY_KEYS: Record = { + 'assembly readiness': 'assembly_readiness', + 'joint design': 'joint_design', + tolerance: 'tolerance', + 'functional adaptation': 'functional_adaptation', + 'usage stability': 'usage_stability', + manufacturability: 'manufacturability', +}; + +export function mapJudgePayload(payload: Record): { + overall: number; + categories: JudgeCategories; + summary: string; +} { + const categories: JudgeCategories = { ...ZERO_CATEGORIES }; + const items = Array.isArray(payload.items) ? payload.items : []; + for (const raw of items) { + const item = raw as { category_en?: unknown; score?: unknown }; + const key = CATEGORY_KEYS[String(item.category_en ?? '').trim().toLowerCase()]; + if (!key) continue; + const score = Number(item.score ?? 0); + categories[key] = Number.isFinite(score) && score >= 0.5 ? 1 : 0; + } + let overall = Number(payload.overall ?? payload.overall_score_normalized ?? 0); + if (!Number.isFinite(overall) || overall <= 0) { + const values = Object.values(categories); + overall = values.reduce((a, b) => a + b, 0) / 6; + } + return { overall, categories, summary: String(payload.summary ?? '') }; +} + +function runWrapper( + args: JudgeCaseArgs, + spawnFn: SpawnFn, +): Promise> { + return new Promise((resolvePromise, reject) => { + const child = spawnFn( + args.pythonBin, + [ + WRAPPER_PY, + '--muse-root', args.museRoot, + '--case-name', args.caseName, + '--case-dir', args.datasetCaseDir, + '--candidate-png', args.candidatePng, + '--out', args.outPath, + '--model', args.model, + '--base-url', args.baseUrl, + ], + { stdio: ['ignore', 'pipe', 'pipe'], timeout: JUDGE_SPAWN_TIMEOUT_MS }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d.toString())); + child.stderr.on('data', (d) => (stderr += d.toString())); + child.on('error', reject); + child.on('close', (code, signal) => { + const line = stdout + .split('\n') + .map((l) => l.trim()) + .reverse() + .find((l) => l.startsWith('{')); + if (!line) { + const suffix = signal !== null && child.killed ? ` (killed after ${JUDGE_SPAWN_TIMEOUT_MS}ms)` : ''; + reject(new Error(`judge wrapper exited ${code}${suffix}: ${stderr.slice(0, 300)}`)); + return; + } + const parsed = JSON.parse(line) as Record; + if (typeof parsed.error === 'string') { + reject(new Error(parsed.error)); + return; + } + resolvePromise(parsed); + }); + }); +} + +export async function judgeCase( + args: JudgeCaseArgs, + stage12Ok: boolean, + spawnFn: SpawnFn = nodeSpawn, +): Promise { + mkdirSync(dirname(args.outPath), { recursive: true }); + if (!stage12Ok) { + const result: JudgeResult = { + overall: 0, + categories: { ...ZERO_CATEGORIES }, + forcedZero: true, + forcedZeroReason: + 'stage 1 sandbox or stage 2 overlap failed; MUSE forces all categories to 0', + }; + writeFileSync(args.outPath, JSON.stringify(result, null, 2)); + return result; + } + const payload = await runWrapper(args, spawnFn); + const mapped = mapJudgePayload(payload); + const result: JudgeResult = { + overall: mapped.overall, + categories: mapped.categories, + forcedZero: false, + summary: mapped.summary, + }; + writeFileSync(args.outPath, JSON.stringify({ ...result, raw: payload }, null, 2)); + return result; +} diff --git a/scripts/musePreflight.test.ts b/scripts/musePreflight.test.ts new file mode 100644 index 000000000..fb3743ad7 --- /dev/null +++ b/scripts/musePreflight.test.ts @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { describe, expect, it } from 'vitest'; +import { runPreflight, formatPreflight, type PreflightDeps } from './musePreflight'; + +function deps(overrides: Partial = {}): PreflightDeps { + return { + env: { + DEEPINFRA_API_KEY: 'x', + MUSE_ROOT: '/muse', + MUSE_PYTHON: '/muse/.venv/bin/python', + KERNELCAD_BIN: './dist/cli/index.js', + }, + exists: () => true, + run: async () => ({ code: 0, stdout: 'ok\n', stderr: '' }), + fetchImpl: (async () => + new Response(JSON.stringify({ data: [{ id: 'google/gemini-3.1-pro' }] }), { + status: 200, + })) as unknown as typeof fetch, + caseCount: () => 106, + ...overrides, + }; +} + +describe('runPreflight', () => { + it('passes when every check is green', async () => { + const report = await runPreflight(deps()); + expect(report.ok).toBe(true); + expect(report.checks.every((c) => c.ok)).toBe(true); + }); + + it('fails when the API key is missing', async () => { + const report = await runPreflight( + deps({ env: { MUSE_ROOT: '/muse', MUSE_PYTHON: '/muse/.venv/bin/python' } }), + ); + expect(report.ok).toBe(false); + expect(report.checks.find((c) => c.name === 'deepinfra key')?.ok).toBe(false); + }); + + it('fails when the judge model is absent from the endpoint', async () => { + const report = await runPreflight( + deps({ + fetchImpl: (async () => + new Response(JSON.stringify({ data: [{ id: 'other' }] }), { + status: 200, + })) as unknown as typeof fetch, + }), + ); + expect(report.checks.find((c) => c.name === 'judge model')?.ok).toBe(false); + }); + + it('formats one line per check', () => { + const text = formatPreflight({ + ok: false, + checks: [ + { name: 'a', ok: true, detail: 'fine' }, + { name: 'b', ok: false, detail: 'broken' }, + ], + }); + expect(text).toContain('PASS a: fine'); + expect(text).toContain('FAIL b: broken'); + }); +}); diff --git a/scripts/musePreflight.ts b/scripts/musePreflight.ts new file mode 100644 index 000000000..b7fad8932 --- /dev/null +++ b/scripts/musePreflight.ts @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { execFile } from 'node:child_process'; +import { existsSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +export interface PreflightCheck { + name: string; + ok: boolean; + detail: string; +} + +export interface PreflightReport { + ok: boolean; + checks: PreflightCheck[]; +} + +export interface PreflightDeps { + env: Record; + exists: (path: string) => boolean; + run: (cmd: string, args: string[]) => Promise<{ code: number; stdout: string; stderr: string }>; + fetchImpl: typeof fetch; + caseCount: () => number; +} + +const JUDGE_MODEL = 'google/gemini-3.1-pro'; +const DEFAULT_BASE_URL = 'https://api.deepinfra.com/v1/openai'; + +function defaultDeps(): PreflightDeps { + const museRoot = process.env.MUSE_ROOT ?? join(process.env.HOME ?? '', 'projects/muse'); + return { + env: process.env as Record, + exists: existsSync, + run: (cmd, args) => + new Promise((res) => { + execFile(cmd, args, { timeout: 120_000 }, (err, stdout, stderr) => { + res({ + code: err ? ((err as { code?: number }).code ?? 1) : 0, + stdout: String(stdout ?? ''), + stderr: String(stderr ?? ''), + }); + }); + }), + fetchImpl: fetch, + caseCount: () => { + const dir = join(museRoot, 'data/muse/cases'); + return existsSync(dir) ? readdirSync(dir).length : 0; + }, + }; +} + +export async function runPreflight(deps: PreflightDeps): Promise { + const checks: PreflightCheck[] = []; + const push = (name: string, ok: boolean, detail: string) => checks.push({ name, ok, detail }); + + const kcadBin = deps.env.KERNELCAD_BIN ?? './dist/cli/index.js'; + const kcadPath = kcadBin.endsWith('.js') ? resolve(kcadBin) : kcadBin; + push('kernelcad cli', deps.exists(kcadPath), kcadPath); + + const museRoot = deps.env.MUSE_ROOT ?? join(deps.env.HOME ?? '', 'projects/muse'); + push('muse checkout', deps.exists(resolve(museRoot, 'src/judge_system')), museRoot); + + const python = deps.env.MUSE_PYTHON ?? join(museRoot, '.venv/bin/python'); + const importCheck = await deps.run(python, [ + '-c', + `import sys; sys.path.insert(0, ${JSON.stringify(resolve(museRoot, 'src'))}); import cadquery, vtk, requests; from judge_system import reverse_pipeline; print('ok')`, + ]); + push( + 'muse python imports', + importCheck.code === 0 && importCheck.stdout.includes('ok'), + importCheck.stderr.trim().slice(0, 200) || python, + ); + + const key = deps.env.DEEPINFRA_API_KEY; + push('deepinfra key', Boolean(key && key.length > 0), key ? 'set' : 'DEEPINFRA_API_KEY missing'); + + const baseUrl = (deps.env.DEEPINFRA_BASE_URL ?? DEFAULT_BASE_URL).replace(/\/+$/, ''); + try { + const resp = await deps.fetchImpl(`${baseUrl}/models`, { + headers: { Authorization: `Bearer ${key ?? ''}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array<{ id?: string }> }; + const ids = (data.data ?? []).map((m) => m.id ?? ''); + push('judge model', resp.ok && ids.includes(JUDGE_MODEL), `${JUDGE_MODEL} @ ${baseUrl}`); + } catch (err) { + push('judge model', false, err instanceof Error ? err.message : String(err)); + } + + const count = deps.caseCount(); + push('muse cases', count === 106, `${count} cases (expected 106)`); + + return { ok: checks.every((c) => c.ok), checks }; +} + +export function formatPreflight(report: PreflightReport): string { + return report.checks.map((c) => `${c.ok ? 'PASS' : 'FAIL'} ${c.name}: ${c.detail}`).join('\n'); +} + +async function main(): Promise { + const report = await runPreflight(defaultDeps()); + console.log(formatPreflight(report)); + process.exit(report.ok ? 0 : 1); +} + +const isEntrypoint = (() => { + if (!process.argv[1]) return false; + try { + return import.meta.url === new URL(`file://${process.argv[1]}`).href; + } catch { + return false; + } +})(); + +if (isEntrypoint) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/scripts/museReport.test.ts b/scripts/museReport.test.ts new file mode 100644 index 000000000..23d81f03a --- /dev/null +++ b/scripts/museReport.test.ts @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { writeReports } from './museReport'; + +function makeCase( + root: string, + name: string, + sandboxOk: boolean, + overlapFree: boolean, + categories?: Record, + phase = 'judged', +) { + const dir = join(root, 'cases', name); + mkdirSync(dir, { recursive: true }); + writeFileSync( + join(dir, 'score.json'), + JSON.stringify({ + gates: { 'evaluates clean': true }, + scored: {}, + gate_pass: true, + score: 1, + attempts: 2, + tokens: { input: 100, output: 20, total: 120 }, + time_ms: 1000, + metrics: { muse_sandbox_ok: sandboxOk, muse_overlap_free: overlapFree }, + }), + ); + writeFileSync( + join(dir, 'state.json'), + JSON.stringify({ phase, attempts: 2, tokens: { in: 100, out: 20 }, protocol: 'muse-v1', updatedAt: 'x' }), + ); + if (categories) { + writeFileSync( + join(dir, 'judge.json'), + JSON.stringify({ overall: 0.5, categories, forcedZero: false }), + ); + } +} + +const CATS = { + assembly_readiness: 1, + joint_design: 0, + tolerance: 1, + functional_adaptation: 1, + usage_stability: 0, + manufacturability: 1, +}; + +describe('writeReports', () => { + it('writes leaderboard json/csv, summary and protocol', () => { + const root = mkdtempSync(join(tmpdir(), 'report-')); + makeCase(root, 'a', true, true, CATS); + makeCase(root, 'b', false, false, CATS); + writeFileSync( + join(root, 'run.json'), + JSON.stringify({ + model: 'm+kcad', + judgeModel: 'judge', + protocol: 'muse-v1', + skills: ['kernelcad'], + workers: 6, + temperature: 0.2, + }), + ); + + writeReports(root); + + const row = JSON.parse(readFileSync(join(root, 'leaderboard.json'), 'utf8')); + expect(row.rows[0].cases).toBe(2); + expect(row.rows[0].judged).toBe(1); + expect(row.rows[0].final).toBe(33.33); + expect(row.rows[0].geom_valid).toBeNull(); + expect(row.validator_status).toBe('unpublished'); + expect(readFileSync(join(root, 'summary.md'), 'utf8')).toContain('Forced-zero'); + expect(readFileSync(join(root, 'protocol.md'), 'utf8')).toContain('validator'); + expect(readFileSync(join(root, 'leaderboard.csv'), 'utf8')).toContain('m+kcad + kernelCAD'); + }); + + it('excludes infra cases from denominators and lists them', () => { + const root = mkdtempSync(join(tmpdir(), 'report-')); + makeCase(root, 'a', true, true, CATS); + const bad = join(root, 'cases', 'bad'); + mkdirSync(bad, { recursive: true }); + writeFileSync( + join(bad, 'state.json'), + JSON.stringify({ phase: 'infra_error', attempts: 0, tokens: { in: 0, out: 0 }, protocol: 'muse-v1', updatedAt: 'x' }), + ); + writeFileSync(join(root, 'run.json'), JSON.stringify({ model: 'm+kcad' })); + + writeReports(root); + + const leaderboard = JSON.parse(readFileSync(join(root, 'leaderboard.json'), 'utf8')); + expect(leaderboard.n_cases).toBe(2); + expect(leaderboard.evaluated_cases).toBe(1); + expect(leaderboard.infra_cases).toBe(1); + expect(leaderboard.rows[0].final).toBe(66.67); + expect(readFileSync(join(root, 'summary.md'), 'utf8')).toContain('Infra errors'); + expect(readFileSync(join(root, 'summary.md'), 'utf8')).toContain('- bad'); + }); +}); diff --git a/scripts/museReport.ts b/scripts/museReport.ts new file mode 100644 index 000000000..6d0dc3d41 --- /dev/null +++ b/scripts/museReport.ts @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + aggregateMuseSamples, + ZERO_CATEGORIES, + type JudgeCategories, + type MuseSample, +} from '../eval/lib/museAggregate'; +import type { Score } from '../eval/types'; + +export interface RunEnvelope { + runId?: string; + model: string; + gitSha?: string; + gitDirty?: boolean; + judgeModel?: string; + baseUrl?: string; + protocol?: string; + skills?: string[]; + workers?: number; + temperature?: number; + caseCount?: number; + tokens?: { tokensIn?: number; tokensOut?: number }; + wallMs?: number; +} + +interface CaseArtifact { + case: string; + infra: boolean; + sandboxOk: boolean; + overlapFree: boolean; + categories?: JudgeCategories; + forcedZero: boolean; + score?: Score; + firstFailureCode?: string; +} + +function loadCase(root: string, name: string): CaseArtifact { + const dir = join(root, 'cases', name); + const statePath = join(dir, 'state.json'); + const state = existsSync(statePath) + ? (JSON.parse(readFileSync(statePath, 'utf8')) as { phase?: string }) + : undefined; + const score: Score | undefined = existsSync(join(dir, 'score.json')) + ? (JSON.parse(readFileSync(join(dir, 'score.json'), 'utf8')) as Score) + : undefined; + const metrics = (score?.metrics ?? {}) as Record; + let categories: JudgeCategories | undefined; + let forcedZero = false; + if (existsSync(join(dir, 'judge.json'))) { + const judge = JSON.parse(readFileSync(join(dir, 'judge.json'), 'utf8')) as { + categories?: JudgeCategories; + forcedZero?: boolean; + }; + categories = { ...ZERO_CATEGORIES, ...(judge.categories ?? {}) }; + forcedZero = judge.forcedZero === true; + } + return { + case: name, + infra: state?.phase === 'infra_error', + sandboxOk: metrics.muse_sandbox_ok === true, + overlapFree: metrics.muse_overlap_free === true, + categories, + forcedZero, + score, + firstFailureCode: score?.firstFailureCode, + }; +} + +function toCsv(rows: Array>): string { + if (rows.length === 0) return ''; + const header = Object.keys(rows[0]); + const cell = (v: string | number | null | undefined): string => + v === null || v === undefined ? '' : String(v); + const lines = [header.join(',')]; + for (const row of rows) { + lines.push(header.map((h) => cell(row[h])).join(',')); + } + return `${lines.join('\n')}\n`; +} + +export function writeReports(runRoot: string): void { + const run: RunEnvelope = existsSync(join(runRoot, 'run.json')) + ? (JSON.parse(readFileSync(join(runRoot, 'run.json'), 'utf8')) as RunEnvelope) + : { model: 'unknown' }; + const casesDir = join(runRoot, 'cases'); + const names = existsSync(casesDir) ? readdirSync(casesDir).sort() : []; + const artifacts = names.map((n) => loadCase(runRoot, n)); + + const samples: MuseSample[] = artifacts.map((a) => ({ + case: a.case, + infra: a.infra, + sandboxOk: a.sandboxOk, + overlapFree: a.overlapFree, + categories: a.categories, + })); + const { row, forcedZeroCases, infraCases } = aggregateMuseSamples(samples, { + model: `${run.model} + kernelCAD`, + }); + + const leaderboard = { + judge: run.judgeModel ?? 'google/gemini-3.1-pro', + n_cases: artifacts.length, + evaluated_cases: row.cases, + infra_cases: infraCases.length, + updated: new Date().toISOString(), + validator_status: 'unpublished', + rows: [row], + }; + writeFileSync(join(runRoot, 'leaderboard.json'), JSON.stringify(leaderboard, null, 2)); + writeFileSync( + join(runRoot, 'leaderboard.csv'), + toCsv([row as unknown as Record]), + ); + + const failureCounts = new Map(); + for (const a of artifacts) { + const code = a.firstFailureCode ?? 'none'; + failureCounts.set(code, (failureCounts.get(code) ?? 0) + 1); + } + const failureLines = [...failureCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([code, count]) => `- ${code}: ${count}`); + + const summary = [ + `# MUSE 106 sweep summary — ${run.model}`, + '', + `Cases attempted: ${artifacts.length} | evaluated (denominator): ${row.cases} | judged: ${row.judged} | forced-zero: ${forcedZeroCases.length} | infra errors: ${infraCases.length}`, + `Sandbox pass: ${row.sandbox}% | Overlap-free: ${row.overlap_free}% | Final: ${row.final}`, + '', + '## Pillars', + '', + `- Functionality: ${row.functionality}`, + `- Manufacturability: ${row.manufacturability}`, + `- Assemblability: ${row.assemblability}`, + '', + '## First failure codes', + '', + ...(failureLines.length > 0 ? failureLines : ['- none']), + '', + '## Forced-zero cases', + '', + ...(forcedZeroCases.length > 0 ? forcedZeroCases.map((c) => `- ${c}`) : ['- none']), + '', + '## Infra errors (excluded from denominators)', + '', + ...(infraCases.length > 0 ? infraCases.map((c) => `- ${c}`) : ['- none']), + '', + ].join('\n'); + writeFileSync(join(runRoot, 'summary.md'), summary); + + const protocol = [ + '# Sweep protocol and deviations', + '', + `- Driver: ${run.model}; protocol version ${run.protocol ?? 'muse-v1'}; temperature ${run.temperature ?? 0.2}; skills: ${(run.skills ?? []).join(', ') || 'n/a'}.`, + `- kernelCAD commit: ${run.gitSha ?? 'unknown'}${run.gitDirty ? ' (dirty tree)' : ''}.`, + '- Generation: kernelCAD product loop — 1 sample per case, up to 2 diagnostic-driven repairs, candidates=1.', + '- Geometry submission: MUSE sandbox runs a CadQuery shim importing a kernelCAD-exported STEP (no CadQuery authored by kernelCAD).', + '- Judge: MUSE `generate_score_sp` + `_run_alignment_judge`, model google/gemini-3.1-pro served via DeepInfra (upstream default serving is OpenRouter preview).', + '- Candidate image: MUSE VTK render for all 106 cases; upstream uses DrawCAD 4-view PNGs for the 97 non-render-only cases (DrawCAD unpublished).', + '- Stage 2: MUSE external `validator` module is unpublished; watertight/manifold/self-intersection and the official `geom_valid` column cannot be computed locally. Overlap-free is computed with MUSE code. Forced-zero locally covers sandbox and overlap only.', + '- Aggregation mirrors upstream `generate_latex_tables_gemini.py@547a724^`; validator columns are null in leaderboard.json.', + '', + ].join('\n'); + writeFileSync(join(runRoot, 'protocol.md'), protocol); +} diff --git a/scripts/runMuseSweep.ts b/scripts/runMuseSweep.ts new file mode 100644 index 000000000..b38286bef --- /dev/null +++ b/scripts/runMuseSweep.ts @@ -0,0 +1,483 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { execFile, execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { MockAgentClient } from '../eval/agent'; +import { OpenAICompatAgentClient } from '../eval/agentOpenAICompat'; +import { generateCase, scoreCase } from '../eval/runner'; +import { buildSystemPrompt, SWEEP_SKILLS } from '../eval/lib/systemPrompt'; +import { mapPool } from '../eval/lib/pool'; +import { isAtLeast, readState, writeState, type MusePhase } from '../eval/lib/museState'; +import { judgeCase } from './museJudge'; +import { runPreflight, formatPreflight, type PreflightReport } from './musePreflight'; +import { writeReports } from './museReport'; +import type { AgentClient, AgentResponse, TaskResult, TranscriptEvent } from '../eval/types'; + +const TASKS_DIR = resolve('eval/tasks'); +const RUNS_DIR = resolve('eval/runs'); +const DEFAULT_MODEL = 'deepseek-ai/DeepSeek-V4.1-Flash'; +const DEFAULT_BASE_URL = 'https://api.deepinfra.com/v1/openai'; +const JUDGE_MODEL = 'google/gemini-3.1-pro'; +const PROTOCOL = 'muse-v1'; + +interface SweepConfig { + runId: string; + runRoot: string; + cases: string[]; + workers: number; + model: string; + baseUrl: string; + temperature: number; + maxAttempts: number; + maxTokens: number; + skills: string[]; + skipJudge: boolean; + force: Set; + maxTokensIn: number; + mockFixture?: string; + startedAt: string; + gitSha: string; + gitDirty: boolean; + env: Record; +} + +export interface CaseOutcome { + case: string; + status: 'ok' | 'skipped' | 'stopped' | 'budget' | 'infra'; + phase?: MusePhase; + finalScore?: number; + attempts?: number; + timeMs?: number; + error?: string; +} + +function parseArgs(argv: string[]): SweepConfig { + const flagValue = (name: string): string | undefined => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; + }; + const has = (name: string): boolean => argv.includes(name); + const list = (name: string): string[] => { + const v = flagValue(name); + return v ? v.split(',').map((s) => s.trim()).filter(Boolean) : []; + }; + /** Collects all non-flag tokens after `name`; each may be comma-separated. */ + const multiList = (name: string): string[] => { + const i = argv.indexOf(name); + if (i < 0) return []; + const out: string[] = []; + for (let j = i + 1; j < argv.length && !argv[j].startsWith('--'); j++) { + out.push(...argv[j].split(',').map((s) => s.trim()).filter(Boolean)); + } + return out; + }; + const fail = (msg: string): never => { + console.error(`ERROR: ${msg}`); + process.exit(1); + }; + + const model = flagValue('--model') ?? DEFAULT_MODEL; + const slug = model + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + let sha = 'nogit'; + let dirty = ''; + try { + sha = execFileSync('git', ['rev-parse', '--short=7', 'HEAD'], { encoding: 'utf8' }).trim(); + dirty = + execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim().length > 0 + ? '-dirty' + : ''; + } catch { + // not a git checkout — leave as nogit + } + const ts = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-'); + const runId = flagValue('--run-id') ?? `muse106-${sha}${dirty}-${slug}-${ts}`; + + const workers = Number(flagValue('--workers') ?? 6); + if (!Number.isInteger(workers) || workers < 1) fail(`--workers must be an integer >= 1, got ${workers}`); + const temperature = Number(flagValue('--temperature') ?? 0.2); + if (!Number.isFinite(temperature)) fail(`--temperature must be a number, got ${temperature}`); + const maxAttempts = Number(flagValue('--max-attempts') ?? 3); + if (!Number.isInteger(maxAttempts) || maxAttempts < 1) fail(`--max-attempts must be an integer >= 1`); + const maxTokens = Number(flagValue('--max-tokens') ?? 8000); + if (!Number.isInteger(maxTokens) || maxTokens < 1) fail(`--max-tokens must be an integer >= 1`); + const maxTokensIn = Number(flagValue('--max-tokens-in') ?? 25_000_000); + if (!Number.isFinite(maxTokensIn)) fail(`--max-tokens-in must be a number`); + + const skills = flagValue('--skills') !== undefined ? list('--skills') : [...SWEEP_SKILLS]; + if (skills.length === 0) fail('--skills must select at least one skill'); + + return { + runId, + runRoot: join(RUNS_DIR, runId), + cases: multiList('--cases'), + workers, + model, + baseUrl: flagValue('--base-url') ?? DEFAULT_BASE_URL, + temperature, + maxAttempts, + maxTokens, + skills, + skipJudge: has('--skip-judge'), + force: new Set(multiList('--force')), + maxTokensIn, + mockFixture: flagValue('--mock-fixture'), + startedAt: new Date().toISOString().replace(/\..+$/, '').replace(/:/g, '-'), + gitSha: sha, + gitDirty: dirty.length > 0, + env: process.env as Record, + }; +} + +function discoverCases(filter: string[]): string[] { + const all = readdirSync(TASKS_DIR) + .filter((name) => name.startsWith('muse-')) + .filter((name) => { + const dir = join(TASKS_DIR, name); + return ( + statSync(dir).isDirectory() && + existsSync(join(dir, 'prompt.md')) && + existsSync(join(dir, 'harness.ts')) + ); + }) + .map((name) => name.replace(/^muse-/, '')); + if (filter.length === 0) return all.sort(); + const wanted = new Set(filter); + const unknown = filter.filter((c) => !all.includes(c)); + if (unknown.length > 0) console.error(`WARN: unknown cases ignored: ${unknown.join(', ')}`); + return all.filter((c) => wanted.has(c)).sort(); +} + +function readScoreResult(caseDir: string): TaskResult | null { + const path = join(caseDir, 'score.json'); + if (!existsSync(path)) return null; + return { + task: caseDir.split('/').pop() ?? 'unknown', + score: JSON.parse(readFileSync(path, 'utf8')), + }; +} + +/** The no-script placeholder written by generateCase. */ +function isNoScriptArtifact(outputScriptPath: string): boolean { + if (!existsSync(outputScriptPath)) return false; + return readFileSync(outputScriptPath, 'utf8').startsWith('// (no script extracted'); +} + +function readCachedAgentResponses(fixturePath: string): AgentResponse[] { + const data = JSON.parse(readFileSync(fixturePath, 'utf8')) as { responses: AgentResponse[] }; + return data.responses; +} + +function sleep(ms: number): Promise { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +async function judgeWithRetries( + args: Parameters[0], + stage12Ok: boolean, + attempts = 5, +): Promise>> { + let lastErr: Error | undefined; + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt > 0) { + const backoff = Math.min(30_000, 1000 * 2 ** (attempt - 1)); + await sleep(backoff + Math.random() * backoff * 0.25); + } + try { + return await judgeCase(args, stage12Ok); + } catch (err) { + lastErr = err instanceof Error ? err : new Error(String(err)); + } + } + throw lastErr ?? new Error('judge failed after retries'); +} + +async function runOneCase( + caseName: string, + cfg: SweepConfig, + agent: AgentClient, + skillMd: string, + totals: { tokensIn: number; tokensOut: number }, +): Promise { + const taskDir = join(TASKS_DIR, `muse-${caseName}`); + const caseDir = join(cfg.runRoot, 'cases', caseName); + mkdirSync(caseDir, { recursive: true }); + + if (existsSync(join(cfg.runRoot, 'STOP'))) { + return { case: caseName, status: 'stopped' }; + } + if (totals.tokensIn >= cfg.maxTokensIn) { + return { case: caseName, status: 'budget' }; + } + + const targetPhase: MusePhase = cfg.skipJudge ? 'scored' : 'judged'; + let state = readState(caseDir); + if (!cfg.force.has(caseName) && state !== null && isAtLeast(state.phase, targetPhase)) { + return { case: caseName, status: 'skipped', phase: state.phase }; + } + + try { + const outputScriptPath = join(caseDir, 'output.kcad.ts'); + let result: TaskResult; + let generationMs = state?.generationMs ?? 0; + let generationEvents: TranscriptEvent[] | undefined; + + const canReuseGeneration = + !cfg.force.has(caseName) && + state !== null && + isAtLeast(state.phase, 'generated') && + existsSync(outputScriptPath); + + if (!canReuseGeneration) { + const gen = await generateCase({ + taskDir, + runDir: caseDir, + agent, + model: cfg.model, + skillMd, + startedAt: cfg.startedAt, + candidates: 1, + maxAttempts: cfg.maxAttempts, + maxTokens: cfg.maxTokens, + temperature: cfg.temperature, + }); + generationMs = gen.timeMs; + generationEvents = gen.events; + totals.tokensIn += gen.tokensIn; + totals.tokensOut += gen.tokensOut; + writeState(caseDir, { + phase: 'generated', + attempts: gen.attempts, + tokens: { in: gen.tokensIn, out: gen.tokensOut }, + firstFailureCode: gen.firstFailureCode, + generationMs: gen.timeMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + state = readState(caseDir); + } + + const cached = readScoreResult(caseDir); + if (cached !== null && !cfg.force.has(caseName) && isAtLeast(state!.phase, 'scored')) { + result = cached; + } else { + result = await scoreCase({ + taskDir, + runDir: caseDir, + outputScriptPath, + events: generationEvents, + attempts: state?.attempts ?? 1, + tokensIn: state?.tokens.in ?? 0, + tokensOut: state?.tokens.out ?? 0, + generationMs, + startedAt: cfg.startedAt, + model: cfg.model, + firstFailureCode: state?.firstFailureCode, + noScript: isNoScriptArtifact(outputScriptPath), + }); + writeState(caseDir, { + phase: 'scored', + attempts: state?.attempts ?? 1, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: result.score?.firstFailureCode ?? state?.firstFailureCode, + generationMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + } + + if (!cfg.skipJudge) { + const metrics = result.score?.metrics ?? {}; + const stage12Ok = metrics.muse_sandbox_ok === true && metrics.muse_overlap_free === true; + const candidatePng = String(metrics.muse_render_png ?? ''); + const museRoot = cfg.env.MUSE_ROOT ?? join(cfg.env.HOME ?? '', 'projects/muse'); + const datasetCaseDir = join(museRoot, 'data/muse/cases', caseName); + const pythonBin = cfg.env.MUSE_PYTHON ?? join(museRoot, '.venv/bin/python'); + + const judge = await judgeWithRetries( + { + caseName, + datasetCaseDir, + candidatePng, + outPath: join(caseDir, 'judge.json'), + museRoot, + pythonBin, + baseUrl: cfg.baseUrl, + model: JUDGE_MODEL, + }, + stage12Ok && candidatePng.length > 0, + ); + writeState(caseDir, { + phase: 'judged', + attempts: state?.attempts ?? 1, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: result.score?.firstFailureCode ?? state?.firstFailureCode, + generationMs, + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + return { + case: caseName, + status: 'ok', + phase: 'judged', + finalScore: judge.overall, + attempts: result.score?.attempts, + timeMs: result.score?.time_ms, + }; + } + + return { + case: caseName, + status: 'ok', + phase: 'scored', + finalScore: result.score?.score, + attempts: result.score?.attempts, + timeMs: result.score?.time_ms, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + writeState(caseDir, { + phase: 'infra_error', + attempts: state?.attempts ?? 0, + tokens: state?.tokens ?? { in: 0, out: 0 }, + firstFailureCode: state?.firstFailureCode, + generationMs: state?.generationMs, + error: message.slice(0, 1000), + protocol: PROTOCOL, + updatedAt: new Date().toISOString(), + }); + return { case: caseName, status: 'infra', error: message }; + } +} + +async function main(): Promise { + const cfg = parseArgs(process.argv.slice(2)); + const preflightOnly = process.argv.includes('--preflight-only'); + + const report: PreflightReport = await runPreflight({ + env: cfg.env, + exists: existsSync, + run: (cmd, args) => + new Promise((res) => { + execFile(cmd, args, { timeout: 120_000 }, (err, stdout, stderr) => { + res({ + code: err ? ((err as { code?: number }).code ?? 1) : 0, + stdout: String(stdout ?? ''), + stderr: String(stderr ?? ''), + }); + }); + }), + fetchImpl: fetch, + caseCount: () => { + const dir = join(cfg.env.MUSE_ROOT ?? join(cfg.env.HOME ?? '', 'projects/muse'), 'data/muse/cases'); + return existsSync(dir) ? readdirSync(dir).length : 0; + }, + }); + console.log(formatPreflight(report)); + const ignoredForMock = new Set(['deepinfra key', 'judge model']); + const blocking = report.checks.filter( + (c) => !(cfg.mockFixture !== undefined && ignoredForMock.has(c.name)), + ); + if (!blocking.every((c) => c.ok)) process.exit(1); + if (preflightOnly) { + console.log('preflight-only: OK'); + return; + } + + const cases = discoverCases(cfg.cases); + if (cases.length === 0) { + console.error('No muse-* tasks found. Run the importer first (plan Task 0).'); + process.exit(1); + } + mkdirSync(cfg.runRoot, { recursive: true }); + + const skillMd = buildSystemPrompt(cfg.skills); + const agent: AgentClient = cfg.mockFixture + ? new MockAgentClient(readCachedAgentResponses(cfg.mockFixture)) + : new OpenAICompatAgentClient({ + baseUrl: cfg.baseUrl, + apiKey: cfg.env.DEEPINFRA_API_KEY ?? '', + }); + + const totals = { tokensIn: 0, tokensOut: 0 }; + const startedAtMs = Date.now(); + const envelope = { + runId: cfg.runId, + model: cfg.model, + baseUrl: cfg.baseUrl, + gitSha: cfg.gitSha, + gitDirty: cfg.gitDirty, + temperature: cfg.temperature, + maxAttempts: cfg.maxAttempts, + maxTokens: cfg.maxTokens, + skills: cfg.skills, + workers: cfg.workers, + protocol: PROTOCOL, + judgeModel: JUDGE_MODEL, + judgeBaseUrl: cfg.baseUrl, + startedAt: cfg.startedAt, + caseCount: cases.length, + }; + writeFileSync(join(cfg.runRoot, 'run.json'), JSON.stringify(envelope, null, 2)); + + const outcomes = await mapPool(cases, cfg.workers, async (caseName, index) => { + const outcome = await runOneCase(caseName, cfg, agent, skillMd, totals); + const done = index + 1; + const badge = outcome.status === 'ok' ? '✓' : outcome.status === 'infra' ? '✗' : '-'; + console.log( + `[${done}/${cases.length}] ${caseName} ${badge} ${ + outcome.finalScore !== undefined ? `final=${outcome.finalScore.toFixed(2)} ` : '' + }${outcome.error ?? outcome.status}`, + ); + return outcome; + }); + + const wallMs = Date.now() - startedAtMs; + const okCount = outcomes.filter((o) => o.status === 'ok').length; + const infra = outcomes.filter((o) => o.status === 'infra'); + writeFileSync( + join(cfg.runRoot, 'run.json'), + JSON.stringify( + { + ...envelope, + finishedAt: new Date().toISOString(), + completed: okCount, + skipped: outcomes.filter((o) => o.status === 'skipped').length, + stopped: outcomes.filter((o) => o.status === 'stopped').length, + budgetSkipped: outcomes.filter((o) => o.status === 'budget').length, + infraErrors: infra.length, + tokens: totals, + wallMs, + outcomes, + }, + null, + 2, + ), + ); + + writeReports(cfg.runRoot); + console.log( + `\n${okCount}/${cases.length} complete in ${(wallMs / 60000).toFixed(1)} min; ${infra.length} infra errors`, + ); + process.exit(infra.length > 0 ? 1 : 0); +} + +const isEntrypoint = (() => { + if (!process.argv[1]) return false; + try { + return import.meta.url === new URL(`file://${process.argv[1]}`).href; + } catch { + return false; + } +})(); + +if (isEntrypoint) { + main().catch((err) => { + console.error('Fatal:', err); + process.exit(1); + }); +}