feat: add evalscope-correctness task for OpenAI-compatible

metainfer/tasks/evalscope_correctness/__init__.py

@@ -0,0 +1,30 @@
+"""evalscope-correctness — evaluate an OpenAI-compatible endpoint with
+EvalScope and report correctness results.
+
+A self-contained task plugin: run EvalScope against a user-supplied,
+already-running OpenAI-compatible chat endpoint, preserve the raw EvalScope
+artifacts under the workspace, and surface a normalized result (per-dataset
+primary score / sample count / optional quality-gate verdict) in the WebUI.
+
+Design notes
+------------
+* This is a **single-run** task — no iteration loop, no shared state graph,
+  no sub-agent pipeline. The "orchestrator" process here is really a thin
+  supervisor that runs EvalScope in an isolated child process per dataset.
+* Execution **completeness** (did EvalScope evaluate every sample without
+  truncation / errors / count mismatch) is tracked separately from the
+  optional **model-quality gate** (minimum per-dataset accuracy/pass@1 the
+  user may configure). A run that completes but fails its quality gate is
+  still a *complete* evaluation (``final_status="success"``); only infra /
+  config / incomplete-result failures surface as ``stopped``. The
+  authoritative pass/fail for quality lives in ``state_dir/result.json``.
+* The API secret is never persisted in ``requirements.json`` — only the name
+  of an environment variable holding the key is stored, and the key is
+  copied only into the child process's environment at run time.
+
+Importing this package registers the orchestrator ``TaskPlugin`` and the
+web ``WebPlugin`` (the canonical single discovery point for new task types).
+"""
+
+from .orchestrator import plugin as _task_plugin  # noqa: F401 — registers TaskPlugin
+from .server import plugin as _web_plugin  # noqa: F401 — registers WebPlugin

metainfer/tasks/evalscope_correctness/form.yaml

@@ -0,0 +1,136 @@
+# Question bank for the `evalscope-correctness` task type.
+#
+# Runs EvalScope correctness benchmarks against an already-running,
+# OpenAI-compatible chat endpoint (e.g. an SGLang server). The task does
+# NOT launch or modify the endpoint — it only sends requests and reports
+# correctness.
+#
+# Notes:
+#   - "file" is not used here; all inputs are text/select/multiselect/number.
+#   - multiselect / select values are the option LABELS as shown below.
+#   - Threshold fields are OPTIONAL per-dataset minimums in [0,1]. A dataset
+#     with no threshold is reported score-only. The quality gate passes only
+#     when every configured threshold passes.
+#   - Only the NAME of an env var holding the API key is stored; the secret
+#     itself is read from the environment at run time and never persisted.
+
+- key: api_url
+  question: "OpenAI-compatible endpoint base URL to evaluate (e.g. http://127.0.0.1:30000/v1):"
+  header: "Endpoint URL"
+  required: true
+  form: text
+
+- key: model
+  question: "Model name the endpoint serves (sent as the OpenAI `model` field):"
+  header: "Model"
+  required: true
+  form: text
+
+- key: model_id
+  question: "Short display id used to name the EvalScope artifact dir (defaults to the model name):"
+  header: "Model id"
+  required: false
+  form: text
+
+- key: benchmarks
+  question: "Which built-in EvalScope correctness benchmarks to run? (select one or more)"
+  header: "Benchmarks"
+  required: true
+  form: multiselect
+  options:
+    - label: "gsm8k"
+      description: "Grade-school math — accuracy (mean)."
+    - label: "gpqa_diamond"
+      description: "Hard graduate-level QA — accuracy (mean)."
+    - label: "humaneval"
+      description: "Code synthesis — pass@1. Requires Docker sandbox execution."
+
+- key: custom_benchmarks
+  question: "Additional EvalScope dataset names supported by this install, comma-separated (e.g. arc, mmlu, race). Evaluated alongside the benchmarks above."
+  header: "Custom datasets"
+  required: false
+  form: text
+
+- key: limit
+  question: "Optional per-subset sample limit for a quick smoke run (blank = full dataset):"
+  header: "Sample limit"
+  required: false
+  form: number
+  default: ""
+
+- key: max_tokens
+  question: "Max generation tokens per request (raise this if any sample is reported truncated):"
+  header: "Max tokens"
+  required: false
+  form: number
+  default: 8192
+
+- key: timeout_seconds
+  question: "Per-request timeout in seconds:"
+  header: "Timeout (s)"
+  required: false
+  form: number
+  default: 300
+
+- key: eval_batch_size
+  question: "Concurrent request batch size (correctness-safe values only):"
+  header: "Batch size"
+  required: false
+  form: select
+  options:
+    - label: "1"
+      description: "Serial — most conservative."
+    - label: "2"
+      description: "Small concurrency (default)."
+    - label: "4"
+      description: "More concurrency."
+
+- key: seed
+  question: "Sampling seed (deterministic, temperature is forced to 0 for correctness):"
+  header: "Seed"
+  required: false
+  form: number
+  default: 42
+
+- key: dataset_cache_dir
+  question: "Optional EvalScope dataset cache dir (blank = EvalScope default):"
+  header: "Dataset cache"
+  required: false
+  form: text
+
+- key: api_key_env_var
+  question: "Name of the environment variable holding the API key (blank = no auth / EvalScope EMPTY). The secret itself is never stored."
+  header: "API key env"
+  required: false
+  form: text
+  default: "EVALSCOPE_API_KEY"
+
+# --- Optional per-dataset minimum-score gates (all in [0,1]) ----------
+# Leave blank to report a dataset's score without gating it.
+
+- key: gate_gsm8k
+  question: "Minimum GSM8K accuracy to pass (0.0-1.0; blank = report only):"
+  header: "GSM8K gate"
+  required: false
+  form: number
+  default: ""
+
+- key: gate_gpqa_diamond
+  question: "Minimum GPQA-Diamond accuracy to pass (0.0-1.0; blank = report only):"
+  header: "GPQA gate"
+  required: false
+  form: number
+  default: ""
+
+- key: gate_humaneval
+  question: "Minimum HumanEval pass@1 to pass (0.0-1.0; blank = report only):"
+  header: "HumanEval gate"
+  required: false
+  form: number
+  default: ""
+
+- key: gate_custom_json
+  question: "Optional JSON map of per-custom-dataset minimums, e.g. {\"arc\": 0.6, \"mmlu\": 0.7}. Ignored if no custom datasets."
+  header: "Custom gates"
+  required: false
+  form: textarea

metainfer/tasks/evalscope_correctness/orchestrator/__init__.py

@@ -0,0 +1,24 @@
+"""evalscope-correctness orchestrator package.
+
+Self-contained supervisor + EvalScope runner. The framework
+(:mod:`metainfer.orchestrator`) never imports this pipeline directly — it
+dispatches to the CLI module declared on :data:`plugin.PLUGIN`.
+
+Layout::
+
+    plugin.py          TaskPlugin descriptor
+    cli.py             ``run <req.json> --state-dir … --workspace-dir …``
+    config.py          parse + validate the evaluation request (from form.yaml)
+    orchestrator.py    supervisor lifecycle: StateStore, PID/signal handling,
+                       per-dataset runner, atomic result.json
+    runner.py          launch an isolated EvalScope child process per dataset
+    evalscope_worker.py  the child: build TaskConfig, run_task, emit a
+                       self-describing ``attempt.json`` (never the secret)
+    report.py          normalize EvalScope reports → result.json (completeness
+                       vs quality-gate separation), all pure + testable
+"""
+
+from metainfer.orchestrator.tasks import register
+from .plugin import PLUGIN
+
+register(PLUGIN)

metainfer/tasks/evalscope_correctness/orchestrator/cli.py

@@ -0,0 +1,46 @@
+"""CLI entry point for the evalscope-correctness orchestrator subprocess.
+
+The launcher spawns::
+
+    python -m metainfer.tasks.evalscope_correctness.orchestrator.cli \\
+        run <requirements.json> --state-dir … --workspace-dir …
+
+Contract required by the framework: ``run`` subcommand + ``--state-dir`` and
+``--workspace-dir`` flags.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="metainfer-evalscope-correctness",
+        description="MetaInfer EvalScope correctness orchestrator.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    run_p = sub.add_parser("run", help="Run the EvalScope correctness evaluation")
+    run_p.add_argument("requirements", type=Path, help="Path to requirements.json")
+    run_p.add_argument("--state-dir", type=Path, default=None,
+                       help="Metadata dir (run.json, timeline.jsonl, result.json).")
+    run_p.add_argument("--workspace-dir", type=Path, default=None,
+                       help="Generated-artifacts dir (raw EvalScope outputs).")
+
+    args = parser.parse_args(argv)
+
+    if args.cmd == "run":
+        from .orchestrator import run_with_requirements
+        return run_with_requirements(
+            requirements_path=args.requirements,
+            state_dir=args.state_dir,
+            workspace_dir=args.workspace_dir,
+        )
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())