demo.py

#!/usr/bin/env python3
"""Run a small CLR example against an external OpenAI-compatible server."""

from clr import run


# Configure the server and model here before running the demo.
BASE_URL = "http://10.195.144.18:8000"
MODEL = "gpt-oss-120b"
API_KEY = "EMPTY"

# A tiny inline dataset for a quick smoke test.
questions = [
    "What is 12 * 12? Give the final answer.",
    "If x + 3 = 10, what is x?",
]
ground_truths = ["144", "7"]

print(f"Loaded {len(questions)} demo questions")

# Run CLR.
metrics = run(
    questions=questions,
    ground_truths=ground_truths,
    base_url=BASE_URL,
    model=MODEL,
    api_key=API_KEY,
    workers=32,
    K=32,
    N=8,
    num_claims=5,
    sampling_params={
        "temperature": 1.0,
        "top_p": 1.0,
        "top_k": 40,
        "max_tokens": 102400,
        # "chat_template_kwargs": {"reasoning_effort": "high"}, # use default setting
    },
    output_path="demo_results.json",
    progress=True,
)

print("\n=== Final Metrics ===")
for key, value in metrics.items():
    if isinstance(value, float):
        if key.startswith("s1_") or key.startswith("pass@") or key.startswith("cons@"):
            print(f"  {key}: {value * 100:.2f}%")
        else:
            print(f"  {key}: {value:.2f}")
    else:
        print(f"  {key}: {value}")