diff --git a/pyproject.toml b/pyproject.toml
index ec90ed2..45dc295 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,7 +23,7 @@ dashboard = [
]
dev = [
"pytest>=8.3,<9",
- "ruff>=0.11,<1",
+ "ruff==0.16.2",
]
[tool.uv]
@@ -42,8 +42,7 @@ addopts = "-ra"
[tool.ruff]
target-version = "py310"
-line-length = 100
+line-length = 88
[tool.ruff.lint]
-select = ["E", "F", "I", "W"]
-ignore = ["E501"]
+select = ["A", "ASYNC", "B", "C4", "DTZ", "E4", "E7", "E9", "F", "FLY", "FURB", "I", "N", "NPY", "PERF", "PGH", "PIE", "PT", "PTH", "RET", "RSE", "RUF", "SIM", "UP", "W605"]
diff --git a/src/conductor_eval/analysis.py b/src/conductor_eval/analysis.py
index 84084e8..ac57917 100644
--- a/src/conductor_eval/analysis.py
+++ b/src/conductor_eval/analysis.py
@@ -1,6 +1,5 @@
import json
import logging
-import os
import sys
from collections import defaultdict
from html import escape
@@ -36,10 +35,10 @@ def apply_plotly_theme(fig):
fig.update_layout(
paper_bgcolor=PLOTLY_BG,
plot_bgcolor=PLOTLY_BG,
- font=dict(color=PLOTLY_TEXT, family="Segoe UI, sans-serif"),
- xaxis=dict(gridcolor=PLOTLY_GRID, zerolinecolor=PLOTLY_GRID),
- yaxis=dict(gridcolor=PLOTLY_GRID, zerolinecolor=PLOTLY_GRID),
- margin=dict(l=60, r=30, t=50, b=60),
+ font={"color": PLOTLY_TEXT, "family": "Segoe UI, sans-serif"},
+ xaxis={"gridcolor": PLOTLY_GRID, "zerolinecolor": PLOTLY_GRID},
+ yaxis={"gridcolor": PLOTLY_GRID, "zerolinecolor": PLOTLY_GRID},
+ margin={"l": 60, "r": 30, "t": 50, "b": 60},
)
return fig
@@ -84,8 +83,13 @@ def _sort_model_names(models):
def _sort_by_model(frame, column="model"):
"""Return a DataFrame ordered by the shared model display-name scheme."""
- order = {model: index for index, model in enumerate(_sort_model_names(frame[column].unique()))}
- return frame.sort_values(column, key=lambda values: values.map(order)).reset_index(drop=True)
+ order = {
+ model: index
+ for index, model in enumerate(_sort_model_names(frame[column].unique()))
+ }
+ return frame.sort_values(column, key=lambda values: values.map(order)).reset_index(
+ drop=True
+ )
def _successful_latency_rows(df):
@@ -132,12 +136,16 @@ def _format_pass_rate_summary(passed, eligible_count, exception_counts):
if not nonzero_exceptions:
exception_summary = "No generation errors"
elif len(nonzero_exceptions) > 1:
- exception_summary = f"{sum(count for _, count in nonzero_exceptions)} run exceptions"
+ exception_summary = (
+ f"{sum(count for _, count in nonzero_exceptions)} run exceptions"
+ )
else:
name, count = nonzero_exceptions[0]
labels = {
"ineligible": "ineligible",
- "generation_error": "generation error" if count == 1 else "generation errors",
+ "generation_error": "generation error"
+ if count == 1
+ else "generation errors",
"rate_limited": "rate limited",
"check_error": "check error" if count == 1 else "check errors",
}
@@ -328,13 +336,16 @@ def compute_scatter_label_layout(
"bottom right": (1, -1),
}
- widths = [min(0.45, max(0.08, (len(label) * 6.4 + 10) / plot_width)) for label in labels]
+ widths = [
+ min(0.45, max(0.08, (len(label) * 6.4 + 10) / plot_width)) for label in labels
+ ]
height = 18 / plot_height
crowding = [
sum(
1
for other in range(len(xs))
- if other != index and (xn[index] - xn[other]) ** 2 + (yn[index] - yn[other]) ** 2 < 0.04
+ if other != index
+ and (xn[index] - xn[other]) ** 2 + (yn[index] - yn[other]) ** 2 < 0.04
)
for index in range(len(xs))
]
@@ -364,7 +375,9 @@ def compute_scatter_label_layout(
alignment = (dx * preferred_x + dy * preferred_y) / magnitude
candidates.append((distance - alignment * 0.002, center_x, center_y))
- candidates.sort(key=lambda candidate: (candidate[0], candidate[2], candidate[1]))
+ candidates.sort(
+ key=lambda candidate: (candidate[0], candidate[2], candidate[1])
+ )
for _, center_x, center_y in candidates:
rectangle = (
center_x - half_width,
@@ -372,7 +385,10 @@ def compute_scatter_label_layout(
center_y - half_height,
center_y + half_height,
)
- if any(_label_rectangles_overlap(rectangle, placed) for placed in placed_rectangles):
+ if any(
+ _label_rectangles_overlap(rectangle, placed)
+ for placed in placed_rectangles
+ ):
continue
placed_rectangles.append(rectangle)
@@ -387,7 +403,9 @@ def compute_scatter_label_layout(
break
if layouts[index] is None:
- raise ValueError("scatter plot is too crowded to place every label without overlap")
+ raise ValueError(
+ "scatter plot is too crowded to place every label without overlap"
+ )
return layouts
@@ -431,7 +449,7 @@ def _add_scatter_labels(
standoff=5,
bgcolor="rgba(26, 26, 46, 0.88)",
borderpad=1,
- font=dict(size=11, color=PLOTLY_TEXT),
+ font={"size": 11, "color": PLOTLY_TEXT},
)
@@ -462,14 +480,14 @@ def load_run(run_path):
# Load config
config_path = run_path / "config.json"
- with open(config_path, "r", encoding="utf-8") as f:
+ with config_path.open(encoding="utf-8") as f:
config = json.load(f)
# Load summary
summary_path = run_path / "summary.json"
summary = {}
if summary_path.exists():
- with open(summary_path, "r", encoding="utf-8") as f:
+ with summary_path.open(encoding="utf-8") as f:
summary = json.load(f)
# Collect all test_results.json files
@@ -480,7 +498,7 @@ def load_run(run_path):
return pd.DataFrame(), config, summary
for tr_path in results_dir.rglob("test_results.json"):
- with open(tr_path, "r", encoding="utf-8") as f:
+ with tr_path.open(encoding="utf-8") as f:
result = json.load(f)
# Result metadata is authoritative; task directory names have no semantics.
@@ -515,12 +533,18 @@ def load_run(run_path):
"temperature": cfg.get("temperature", 0.0),
# Metrics
"api_latency": metrics.get("api_latency"),
- "attempt_latency": metrics.get("attempt_latency", metrics.get("api_latency")),
+ "attempt_latency": metrics.get(
+ "attempt_latency", metrics.get("api_latency")
+ ),
"cost": metrics.get("cost"),
- "cost_available": metrics.get("cost_available", metrics.get("cost") is not None),
+ "cost_available": metrics.get(
+ "cost_available", metrics.get("cost") is not None
+ ),
# Overall
"overall_pass": (
- tests.get("overall_pass", False) if overall_status in {"passed", "failed"} else None
+ tests.get("overall_pass", False)
+ if overall_status in {"passed", "failed"}
+ else None
),
"overall_status": overall_status,
"overall_eligible": overall_status in {"passed", "failed"},
@@ -531,7 +555,9 @@ def load_run(run_path):
"scale_correct": scale_test.get("correct", 0),
"scale_incorrect": scale_test.get("incorrect", 0),
"scale_pitches_correct": scale_test.get("pitches", {}).get("correct", []),
- "scale_pitches_incorrect": scale_test.get("pitches", {}).get("incorrect", []),
+ "scale_pitches_incorrect": scale_test.get("pitches", {}).get(
+ "incorrect", []
+ ),
# Duration test
"duration_ran": duration_test.get("ran", False),
"duration_total": duration_test.get("total", 0),
@@ -541,13 +567,17 @@ def load_run(run_path):
"duration_param": duration_test.get("params", {}).get("duration", ""),
# Texture tests
"monophony_ran": monophony_test.get("ran", False),
- "monophony_eligible": monophony_test.get("eligible", monophony_test.get("ran", False)),
+ "monophony_eligible": monophony_test.get(
+ "eligible", monophony_test.get("ran", False)
+ ),
"monophony_pass": monophony_test.get("passed", False),
"monophony_max_polyphony": monophony_test.get("max_polyphony", 0),
"monophony_distribution": monophony_test.get("polyphony_distribution", {}),
"monophony_percentages": monophony_test.get("polyphony_percentages", {}),
"polyphony_ran": polyphony_test.get("ran", False),
- "polyphony_eligible": polyphony_test.get("eligible", polyphony_test.get("ran", False)),
+ "polyphony_eligible": polyphony_test.get(
+ "eligible", polyphony_test.get("ran", False)
+ ),
"polyphony_pass": polyphony_test.get("passed", False),
"polyphony_max_polyphony": polyphony_test.get("max_polyphony", 0),
"polyphony_min_voices": polyphony_test.get("params", {}).get(
@@ -563,11 +593,17 @@ def load_run(run_path):
# Harmonic rhythm test
"harmonic_rhythm_ran": harmonic_rhythm_test.get("ran", False),
"harmonic_rhythm_pass": harmonic_rhythm_test.get("passed", False),
- "harmonic_rhythm_missing_onsets": harmonic_rhythm_test.get("missing_onsets", []),
- "harmonic_rhythm_unexpected_onsets": harmonic_rhythm_test.get("unexpected_onsets", []),
+ "harmonic_rhythm_missing_onsets": harmonic_rhythm_test.get(
+ "missing_onsets", []
+ ),
+ "harmonic_rhythm_unexpected_onsets": harmonic_rhythm_test.get(
+ "unexpected_onsets", []
+ ),
# Chord event-position test
"chord_event_positions_ran": chord_event_positions_test.get("ran", False),
- "chord_event_positions_pass": chord_event_positions_test.get("passed", False),
+ "chord_event_positions_pass": chord_event_positions_test.get(
+ "passed", False
+ ),
"chord_event_positions_missing": chord_event_positions_test.get(
"missing_positions", []
),
@@ -582,22 +618,30 @@ def load_run(run_path):
# Compute derived columns
df["has_error"] = df["error"].apply(lambda x: isinstance(x, str) and len(x) > 0)
df["scale_accuracy"] = df.apply(
- lambda r: r["scale_correct"] / r["scale_total"] if r["scale_total"] > 0 else None,
+ lambda r: (
+ r["scale_correct"] / r["scale_total"] if r["scale_total"] > 0 else None
+ ),
axis=1,
)
df["duration_accuracy"] = df.apply(
lambda r: (
- r["duration_correct"] / r["duration_total"] if r["duration_total"] > 0 else None
+ r["duration_correct"] / r["duration_total"]
+ if r["duration_total"] > 0
+ else None
),
axis=1,
)
df["scale_pass"] = df.apply(
- lambda r: r["scale_incorrect"] == 0 and r["scale_ran"] and r["scale_total"] > 0,
+ lambda r: (
+ r["scale_incorrect"] == 0 and r["scale_ran"] and r["scale_total"] > 0
+ ),
axis=1,
)
df["duration_pass"] = df.apply(
lambda r: (
- r["duration_incorrect"] == 0 and r["duration_ran"] and r["duration_total"] > 0
+ r["duration_incorrect"] == 0
+ and r["duration_ran"]
+ and r["duration_total"] > 0
),
axis=1,
)
@@ -608,7 +652,9 @@ def load_run(run_path):
"harmonic_rhythm",
"chord_event_positions",
):
- df[f"{test_name}_pass"] = df[f"{test_name}_ran"] & df[f"{test_name}_pass"].fillna(False)
+ df[f"{test_name}_pass"] = df[f"{test_name}_ran"] & df[
+ f"{test_name}_pass"
+ ].fillna(False)
df["polyphony_voice_shortfall"] = df.apply(
lambda r: (
@@ -620,7 +666,11 @@ def load_run(run_path):
)
df["chord_progression_failed_bars"] = df.apply(
lambda r: (
- [bar for bar in r["chord_progression_bars"] if not bar.get("passed", False)]
+ [
+ bar
+ for bar in r["chord_progression_bars"]
+ if not bar.get("passed", False)
+ ]
if r["chord_progression_ran"]
else []
),
@@ -636,7 +686,7 @@ def load_run(run_path):
def _instance_name(row):
if row["effort"] is not None and pd.notna(row["effort"]):
return f"{row['base_model']} ({row['effort']})"
- elif row["use_thinking"]:
+ if row["use_thinking"]:
return f"{row['base_model']} (reasoning)"
return row["base_model"]
@@ -653,7 +703,9 @@ def _instance_name(row):
has_standard = not subset["use_thinking"].all()
if no_effort and has_thinking and has_standard:
std_mask = mask & ~df["use_thinking"]
- df.loc[std_mask, "model"] = df.loc[std_mask, "base_model"] + " (standard)"
+ df.loc[std_mask, "model"] = (
+ df.loc[std_mask, "base_model"] + " (standard)"
+ )
logger.info("Loaded %d results from %s", len(df), run_path)
return df, config, summary
@@ -673,8 +725,9 @@ def list_available_runs(base_dir=None):
base = get_evaluations_dir() if base_dir is None else Path(base_dir)
if not base.exists():
return []
- runs = [d for d in sorted(base.iterdir()) if d.is_dir() and (d / "config.json").exists()]
- return runs
+ return [
+ d for d in sorted(base.iterdir()) if d.is_dir() and (d / "config.json").exists()
+ ]
def select_run_interactive(base_dir=None):
@@ -699,14 +752,14 @@ def select_run_interactive(base_dir=None):
for i, run in enumerate(runs, 1):
# Try to load config for a nice display
try:
- with open(run / "config.json", "r") as f:
+ with (run / "config.json").open() as f:
cfg = json.load(f)
name = cfg.get("run_name", run.name)
ts = cfg.get("timestamp", "")
models = [m[1] if isinstance(m, list) else m for m in cfg.get("models", [])]
model_count = len(models)
print(f" [{i}] {name} ({ts}, {model_count} models)")
- except Exception:
+ except Exception: # noqa: PERF203 - each malformed run should be skipped independently
print(f" [{i}] {run.name}")
print("-" * 60)
@@ -717,7 +770,7 @@ def select_run_interactive(base_dir=None):
if 0 <= idx < len(runs):
return runs[idx]
print(f"Please enter a number between 1 and {len(runs)}")
- except (ValueError, EOFError):
+ except (ValueError, EOFError): # noqa: PERF203 - retrying user input requires this loop
print("Invalid input. Please enter a number.")
@@ -750,9 +803,17 @@ def _rate_labels_and_counts(rates, numerators, denominators):
"""Return consistent rate labels and Plotly customdata count pairs."""
labels = [
f"{rate:.1f}% ({int(numerator)}/{int(denominator)})"
- for rate, numerator, denominator in zip(rates, numerators, denominators)
+ for rate, numerator, denominator in zip(
+ rates, numerators, denominators, strict=False
+ )
]
- counts = list(zip(pd.Series(numerators).astype(int), pd.Series(denominators).astype(int)))
+ counts = list(
+ zip(
+ pd.Series(numerators).astype(int),
+ pd.Series(denominators).astype(int),
+ strict=False,
+ )
+ )
return labels, counts
@@ -795,7 +856,9 @@ def build_pass_rate_by_model(df):
)
stats["pass_rate"] = (stats["passed"] / stats["tested"] * 100).round(1)
stats = _sort_by_model(stats)
- labels, counts = _rate_labels_and_counts(stats["pass_rate"], stats["passed"], stats["tested"])
+ labels, counts = _rate_labels_and_counts(
+ stats["pass_rate"], stats["passed"], stats["tested"]
+ )
fig = go.Figure(
go.Bar(
@@ -806,17 +869,21 @@ def build_pass_rate_by_model(df):
textposition="auto",
customdata=counts,
hovertemplate=_rate_hover_template(
- "Model: %{y}", rate_value="%{x:.1f}", denominator_label="Eligible generations"
+ "Model: %{y}",
+ rate_value="%{x:.1f}",
+ denominator_label="Eligible generations",
),
- marker_color=[MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))],
+ marker_color=[
+ MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))
+ ],
)
)
fig.update_layout(
title="Overall Pass Rate by Model",
xaxis_title="Pass Rate (%)",
yaxis_title="",
- xaxis=dict(range=[0, 105]),
- yaxis=dict(autorange="reversed"),
+ xaxis={"range": [0, 105]},
+ yaxis={"autorange": "reversed"},
)
return apply_plotly_theme(fig)
@@ -831,9 +898,9 @@ def _empty_performance_figure(title, message):
xref="paper",
yref="paper",
showarrow=False,
- font=dict(color=PLOTLY_TEXT, size=14),
+ font={"color": PLOTLY_TEXT, "size": 14},
)
- fig.update_layout(title=title, xaxis=dict(visible=False), yaxis=dict(visible=False))
+ fig.update_layout(title=title, xaxis={"visible": False}, yaxis={"visible": False})
return apply_plotly_theme(fig)
@@ -842,10 +909,16 @@ def _add_check_pass_rate_trace(fig, eligible, pass_column, trace_name):
if eligible.empty:
return
- stats = eligible.groupby("model")[pass_column].agg(tested="count", passed="sum").reset_index()
+ stats = (
+ eligible.groupby("model")[pass_column]
+ .agg(tested="count", passed="sum")
+ .reset_index()
+ )
stats = _sort_by_model(stats)
stats["pass_rate"] = (stats["passed"] / stats["tested"] * 100).round(1)
- labels, counts = _rate_labels_and_counts(stats["pass_rate"], stats["passed"], stats["tested"])
+ labels, counts = _rate_labels_and_counts(
+ stats["pass_rate"], stats["passed"], stats["tested"]
+ )
fig.add_trace(
go.Bar(
name=trace_name,
@@ -868,7 +941,7 @@ def _finish_check_pass_rate_figure(fig, title):
title=title,
xaxis_title="Model",
yaxis_title="Pass Rate (%)",
- yaxis=dict(range=[0, 105]),
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -883,7 +956,9 @@ def build_duration_adherence_by_model(df):
& df["duration_param"].isin(["quarter", "eighth"])
]
if eligible.empty:
- return _empty_performance_figure(title, "No quarter- or eighth-note duration data")
+ return _empty_performance_figure(
+ title, "No quarter- or eighth-note duration data"
+ )
fig = go.Figure()
for duration, label in (("quarter", "Quarter Notes"), ("eighth", "Eighth Notes")):
@@ -909,8 +984,12 @@ def build_texture_performance_by_model(df):
return _empty_performance_figure(title, "No texture checks ran")
fig = go.Figure()
- _add_check_pass_rate_trace(fig, df[monophony_eligible], "monophony_pass", "Monophony")
- _add_check_pass_rate_trace(fig, df[polyphony_eligible], "polyphony_pass", "Polyphony")
+ _add_check_pass_rate_trace(
+ fig, df[monophony_eligible], "monophony_pass", "Monophony"
+ )
+ _add_check_pass_rate_trace(
+ fig, df[polyphony_eligible], "polyphony_pass", "Polyphony"
+ )
return _finish_check_pass_rate_figure(fig, title)
@@ -958,10 +1037,14 @@ def build_model_root_heatmap(df):
pivot = grouped.pivot(index="model", columns="root", values="pass_rate")
pivot = pivot.reindex(_sort_model_names(pivot.index))
passed = (
- grouped.pivot(index="model", columns="root", values="passed").reindex_like(pivot).fillna(0)
+ grouped.pivot(index="model", columns="root", values="passed")
+ .reindex_like(pivot)
+ .fillna(0)
)
tested = (
- grouped.pivot(index="model", columns="root", values="tested").reindex_like(pivot).fillna(0)
+ grouped.pivot(index="model", columns="root", values="tested")
+ .reindex_like(pivot)
+ .fillna(0)
)
counts = [
[
@@ -987,14 +1070,14 @@ def build_model_root_heatmap(df):
colorscale="RdYlGn",
zmin=0,
zmax=100,
- colorbar=dict(title="Pass %"),
+ colorbar={"title": "Pass %"},
)
)
fig.update_layout(
title="Pass Rate: Model x Root",
xaxis_title="Root",
yaxis_title="Model",
- yaxis=dict(autorange="reversed"),
+ yaxis={"autorange": "reversed"},
)
return apply_plotly_theme(fig)
@@ -1024,15 +1107,23 @@ def build_major_vs_minor_by_model(df):
mdf = df[df["model"] == model]
maj = mdf[mdf["scale"] == "major"]
mn = mdf[mdf["scale"] == "minor"]
- major_rates.append(round(maj["overall_pass"].mean() * 100, 1) if len(maj) > 0 else 0)
- minor_rates.append(round(mn["overall_pass"].mean() * 100, 1) if len(mn) > 0 else 0)
+ major_rates.append(
+ round(maj["overall_pass"].mean() * 100, 1) if len(maj) > 0 else 0
+ )
+ minor_rates.append(
+ round(mn["overall_pass"].mean() * 100, 1) if len(mn) > 0 else 0
+ )
major_passed.append(int(maj["overall_pass"].sum()))
minor_passed.append(int(mn["overall_pass"].sum()))
major_tested.append(len(maj))
minor_tested.append(len(mn))
- major_labels, major_counts = _rate_labels_and_counts(major_rates, major_passed, major_tested)
- minor_labels, minor_counts = _rate_labels_and_counts(minor_rates, minor_passed, minor_tested)
+ major_labels, major_counts = _rate_labels_and_counts(
+ major_rates, major_passed, major_tested
+ )
+ minor_labels, minor_counts = _rate_labels_and_counts(
+ minor_rates, minor_passed, minor_tested
+ )
fig = go.Figure()
fig.add_trace(
@@ -1068,7 +1159,7 @@ def build_major_vs_minor_by_model(df):
title="Major vs Minor Pass Rate by Model",
xaxis_title="Model",
yaxis_title="Pass Rate (%)",
- yaxis=dict(range=[0, 105]),
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -1096,7 +1187,9 @@ def build_root_pass_rate(df):
)
stats["pass_rate"] = (stats["passed"] / stats["tested"] * 100).round(1)
stats = stats.sort_values("pass_rate", ascending=False)
- labels, counts = _rate_labels_and_counts(stats["pass_rate"], stats["passed"], stats["tested"])
+ labels, counts = _rate_labels_and_counts(
+ stats["pass_rate"], stats["passed"], stats["tested"]
+ )
fig = go.Figure(
go.Bar(
@@ -1115,7 +1208,7 @@ def build_root_pass_rate(df):
title="Pass Rate by Root Note",
xaxis_title="Root Note",
yaxis_title="Pass Rate (%)",
- yaxis=dict(range=[0, 105]),
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -1145,15 +1238,23 @@ def build_root_scale_grouped(df):
rdf = df[df["root"] == root]
maj = rdf[rdf["scale"] == "major"]
mn = rdf[rdf["scale"] == "minor"]
- major_rates.append(round(maj["overall_pass"].mean() * 100, 1) if len(maj) > 0 else 0)
- minor_rates.append(round(mn["overall_pass"].mean() * 100, 1) if len(mn) > 0 else 0)
+ major_rates.append(
+ round(maj["overall_pass"].mean() * 100, 1) if len(maj) > 0 else 0
+ )
+ minor_rates.append(
+ round(mn["overall_pass"].mean() * 100, 1) if len(mn) > 0 else 0
+ )
major_passed.append(int(maj["overall_pass"].sum()))
minor_passed.append(int(mn["overall_pass"].sum()))
major_tested.append(len(maj))
minor_tested.append(len(mn))
- major_labels, major_counts = _rate_labels_and_counts(major_rates, major_passed, major_tested)
- minor_labels, minor_counts = _rate_labels_and_counts(minor_rates, minor_passed, minor_tested)
+ major_labels, major_counts = _rate_labels_and_counts(
+ major_rates, major_passed, major_tested
+ )
+ minor_labels, minor_counts = _rate_labels_and_counts(
+ minor_rates, minor_passed, minor_tested
+ )
fig = go.Figure()
fig.add_trace(
@@ -1189,7 +1290,7 @@ def build_root_scale_grouped(df):
title="Pass Rate by Root Note (Major vs Minor)",
xaxis_title="Root Note",
yaxis_title="Pass Rate (%)",
- yaxis=dict(range=[0, 105]),
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -1253,14 +1354,14 @@ def build_root_scale_heatmap(df):
colorscale="RdYlGn",
zmin=0,
zmax=100,
- colorbar=dict(title="Pass %"),
+ colorbar={"title": "Pass %"},
)
)
fig.update_layout(
title="Pass Rate: Model x Root+Scale",
xaxis_title="Root + Scale",
yaxis_title="Model",
- yaxis=dict(autorange="reversed"),
+ yaxis={"autorange": "reversed"},
)
return apply_plotly_theme(fig)
@@ -1288,7 +1389,9 @@ def build_latency_box(df):
y=mdf["api_latency"],
name=model,
boxmean=True,
- hovertemplate=("Model: %{fullData.name}
Latency: %{y:.2f}s"),
+ hovertemplate=(
+ "Model: %{fullData.name}
Latency: %{y:.2f}s"
+ ),
)
)
@@ -1345,10 +1448,12 @@ def build_latency_vs_pass(df):
"Model: %{customdata}
Average latency: %{x:.2f}s
"
"Pass rate: %{y:.1f}%"
),
- marker=dict(
- size=stats["latency_count"] / stats["latency_count"].max() * 30 + 10,
- color=[MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))],
- ),
+ marker={
+ "size": stats["latency_count"] / stats["latency_count"].max() * 30 + 10,
+ "color": [
+ MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))
+ ],
+ },
)
)
_add_scatter_labels(
@@ -1363,8 +1468,8 @@ def build_latency_vs_pass(df):
title="Latency vs Pass Rate by Model",
xaxis_title="Average Latency (seconds)",
yaxis_title="Pass Rate (%)",
- xaxis=dict(range=list(x_bounds)),
- yaxis=dict(range=[0, 105]),
+ xaxis={"range": list(x_bounds)},
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -1398,7 +1503,7 @@ def build_cost_by_model(df):
fig.add_annotation(
text="All costs are $0 (local models)",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Cost Analysis")
return apply_plotly_theme(fig)
@@ -1422,7 +1527,7 @@ def build_cost_by_model(df):
title="Total Cost by Model",
xaxis_title="Total Cost ($)",
yaxis_title="",
- yaxis=dict(autorange="reversed"),
+ yaxis={"autorange": "reversed"},
)
return apply_plotly_theme(fig)
@@ -1450,7 +1555,9 @@ def build_cost_vs_pass(df):
.reset_index()
)
pass_stats = (
- eligible_rows.groupby("model").agg(pass_rate=("overall_pass", "mean")).reset_index()
+ eligible_rows.groupby("model")
+ .agg(pass_rate=("overall_pass", "mean"))
+ .reset_index()
)
stats = cost_stats.merge(pass_stats, on="model", how="inner")
if stats.empty:
@@ -1461,7 +1568,7 @@ def build_cost_vs_pass(df):
fig.add_annotation(
text="All costs are $0 (local models)",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Cost vs Pass Rate")
return apply_plotly_theme(fig)
@@ -1481,10 +1588,12 @@ def build_cost_vs_pass(df):
"Model: %{customdata}
Cost per generation: $%{x:.5f}
"
"Pass rate: %{y:.1f}%"
),
- marker=dict(
- size=15,
- color=[MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))],
- ),
+ marker={
+ "size": 15,
+ "color": [
+ MODEL_COLORS[i % len(MODEL_COLORS)] for i in range(len(stats))
+ ],
+ },
)
)
_add_scatter_labels(
@@ -1499,8 +1608,8 @@ def build_cost_vs_pass(df):
title="Cost per Generation vs Pass Rate",
xaxis_title="Cost per Generation ($)",
yaxis_title="Pass Rate (%)",
- xaxis=dict(range=list(x_bounds)),
- yaxis=dict(range=[0, 105]),
+ xaxis={"range": list(x_bounds)},
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -1534,14 +1643,14 @@ def build_incorrect_pitches_by_model(df):
fig.add_annotation(
text="No incorrect pitches found",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Incorrect Pitches by Model")
return apply_plotly_theme(fig)
# Build grouped bar chart
all_notes = sorted(
- set(n for counts in model_pitch_counts.values() for n in counts),
+ {n for counts in model_pitch_counts.values() for n in counts},
key=lambda n: NOTE_NAMES.index(n) if n in NOTE_NAMES else 99,
)
fig = go.Figure()
@@ -1550,8 +1659,11 @@ def build_incorrect_pitches_by_model(df):
values = [counts.get(note, 0) for note in all_notes]
total = sum(values)
customdata = [
- [note_name_to_pitch_class(note), round(count / total * 100, 1) if total else 0]
- for note, count in zip(all_notes, values)
+ [
+ note_name_to_pitch_class(note),
+ round(count / total * 100, 1) if total else 0,
+ ]
+ for note, count in zip(all_notes, values, strict=False)
]
fig.add_trace(
go.Bar(
@@ -1609,7 +1721,7 @@ def build_incorrect_intervals_by_model(df):
fig.add_annotation(
text="No incorrect intervals found",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Incorrect Intervals by Model")
return apply_plotly_theme(fig)
@@ -1627,8 +1739,11 @@ def build_incorrect_intervals_by_model(df):
values = [counts.get(interval, 0) for interval in all_intervals]
total = sum(values)
customdata = [
- [INTERVAL_NAMES.index(interval), round(count / total * 100, 1) if total else 0]
- for interval, count in zip(all_intervals, values)
+ [
+ INTERVAL_NAMES.index(interval),
+ round(count / total * 100, 1) if total else 0,
+ ]
+ for interval, count in zip(all_intervals, values, strict=False)
]
fig.add_trace(
go.Bar(
@@ -1686,12 +1801,14 @@ def build_duration_errors_by_model(df):
fig.add_annotation(
text="No duration errors found",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Duration Errors by Model")
return apply_plotly_theme(fig)
- all_labels = sorted({label for counts in model_dur_counts.values() for label in counts})
+ all_labels = sorted(
+ {label for counts in model_dur_counts.values() for label in counts}
+ )
fig = go.Figure()
for model in _sort_model_names(model_dur_counts):
@@ -1747,17 +1864,21 @@ def build_effort_impact_delta(df):
fig.add_annotation(
text="No effort-level data in this run",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Effort Level Impact")
return apply_plotly_theme(fig)
# Define a canonical effort ordering so we can identify "lowest" and "highest"
- EFFORT_ORDER = ["none", "minimal", "low", "medium", "high", "xhigh", "max"]
+ effort_order = ["none", "minimal", "low", "medium", "high", "xhigh", "max"]
def effort_rank(e):
e_lower = str(e).lower()
- return EFFORT_ORDER.index(e_lower) if e_lower in EFFORT_ORDER else len(EFFORT_ORDER)
+ return (
+ effort_order.index(e_lower)
+ if e_lower in effort_order
+ else len(effort_order)
+ )
# Per (base_model, effort) pass rate
stats = (
@@ -1797,7 +1918,7 @@ def effort_rank(e):
fig.add_annotation(
text="No models with multiple effort levels",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Effort Level Impact")
return apply_plotly_theme(fig)
@@ -1818,6 +1939,7 @@ def effort_rank(e):
result["lowest_effort"],
result["highest_rate"],
result["highest_effort"],
+ strict=False,
)
],
textposition="auto",
@@ -1829,6 +1951,7 @@ def effort_rank(e):
result["highest_effort"],
result["highest_rate"],
result["highest_count"],
+ strict=False,
)
),
hovertemplate=(
@@ -1882,7 +2005,7 @@ def build_reasoning_toggle_comparison(df):
fig.add_annotation(
text="No toggle-based reasoning models in this run",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Standard vs Reasoning Toggle")
return apply_plotly_theme(fig)
@@ -1906,13 +2029,19 @@ def build_reasoning_toggle_comparison(df):
std_eligible = _eligible_overall_rows(std)
reas_eligible = _eligible_overall_rows(reas)
std_rates.append(
- round(std_eligible["overall_pass"].mean() * 100, 1) if not std_eligible.empty else 0
+ round(std_eligible["overall_pass"].mean() * 100, 1)
+ if not std_eligible.empty
+ else 0
)
reas_rates.append(
- round(reas_eligible["overall_pass"].mean() * 100, 1) if not reas_eligible.empty else 0
+ round(reas_eligible["overall_pass"].mean() * 100, 1)
+ if not reas_eligible.empty
+ else 0
)
std_latencies.append(round(std["api_latency"].mean(), 1) if len(std) > 0 else 0)
- reas_latencies.append(round(reas["api_latency"].mean(), 1) if len(reas) > 0 else 0)
+ reas_latencies.append(
+ round(reas["api_latency"].mean(), 1) if len(reas) > 0 else 0
+ )
std_costs.append(round(std["cost"].mean(), 5) if len(std) > 0 else 0)
reas_costs.append(round(reas["cost"].mean(), 5) if len(reas) > 0 else 0)
std_counts.append(len(std_eligible))
@@ -1938,7 +2067,7 @@ def build_reasoning_toggle_comparison(df):
orientation="h",
text=[f"{v}%" for v in std_rates],
textposition="auto",
- customdata=list(zip(std_passed, std_counts)),
+ customdata=list(zip(std_passed, std_counts, strict=False)),
hovertemplate=_rate_hover_template(
"Model: %{y}
Mode: Standard",
rate_value="%{x:.1f}",
@@ -1958,7 +2087,7 @@ def build_reasoning_toggle_comparison(df):
orientation="h",
text=[f"{v}%" for v in reas_rates],
textposition="auto",
- customdata=list(zip(reas_passed, reas_counts)),
+ customdata=list(zip(reas_passed, reas_counts, strict=False)),
hovertemplate=_rate_hover_template(
"Model: %{y}
Mode: Reasoning",
rate_value="%{x:.1f}",
@@ -2097,7 +2226,7 @@ def build_reasoning_cost_effectiveness(df):
fig.add_annotation(
text="All costs are $0 (local models)",
showarrow=False,
- font=dict(size=16, color=PLOTLY_TEXT),
+ font={"size": 16, "color": PLOTLY_TEXT},
)
fig.update_layout(title="Reasoning Cost-Effectiveness")
return apply_plotly_theme(fig)
@@ -2109,7 +2238,9 @@ def build_reasoning_cost_effectiveness(df):
# Assign colors per base_model
base_models = _sort_model_names(stats["base_model"].unique())
- color_map = {bm: MODEL_COLORS[i % len(MODEL_COLORS)] for i, bm in enumerate(base_models)}
+ color_map = {
+ bm: MODEL_COLORS[i % len(MODEL_COLORS)] for i, bm in enumerate(base_models)
+ }
fig = go.Figure()
@@ -2123,7 +2254,7 @@ def build_reasoning_cost_effectiveness(df):
x=bm_stats["cost_per_gen"],
y=bm_stats["pass_rate_pct"],
mode="lines",
- line=dict(color=color_map[base], width=1.5, dash="dot"),
+ line={"color": color_map[base], "width": 1.5, "dash": "dot"},
showlegend=False,
hoverinfo="skip",
)
@@ -2144,6 +2275,7 @@ def build_reasoning_cost_effectiveness(df):
bm_stats["passed"].astype(int),
bm_stats["eligible"].astype(int),
bm_stats["costed"].astype(int),
+ strict=False,
)
),
hovertemplate=(
@@ -2152,7 +2284,7 @@ def build_reasoning_cost_effectiveness(df):
"Eligible generations: %{customdata[2]}
"
"Costed generations: %{customdata[3]}"
),
- marker=dict(size=12, color=color_map[base]),
+ marker={"size": 12, "color": color_map[base]},
)
)
@@ -2168,8 +2300,8 @@ def build_reasoning_cost_effectiveness(df):
title="Reasoning Cost-Effectiveness (Cost per Generation vs Pass Rate)",
xaxis_title="Cost per Generation ($)",
yaxis_title="Pass Rate (%)",
- xaxis=dict(range=list(x_bounds)),
- yaxis=dict(range=[0, 105]),
+ xaxis={"range": list(x_bounds)},
+ yaxis={"range": [0, 105]},
)
return apply_plotly_theme(fig)
@@ -2196,7 +2328,9 @@ def build_failure_rate_by_model(df):
)
stats["error_rate"] = (stats["errors"] / stats["total"] * 100).round(1)
stats = _sort_by_model(stats)
- labels, counts = _rate_labels_and_counts(stats["error_rate"], stats["errors"], stats["total"])
+ labels, counts = _rate_labels_and_counts(
+ stats["error_rate"], stats["errors"], stats["total"]
+ )
fig = go.Figure(
go.Bar(
@@ -2220,8 +2354,8 @@ def build_failure_rate_by_model(df):
title="Generation Failure Rate by Model",
xaxis_title="Failure Rate (%)",
yaxis_title="",
- xaxis=dict(range=[0, max(stats["error_rate"].max() * 1.2, 10)]),
- yaxis=dict(autorange="reversed"),
+ xaxis={"range": [0, max(stats["error_rate"].max() * 1.2, 10)]},
+ yaxis={"autorange": "reversed"},
)
return apply_plotly_theme(fig)
@@ -2291,7 +2425,9 @@ def build_filter_bar(df):
[
dbc.Col(
[
- html.Label("Models", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}),
+ html.Label(
+ "Models", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}
+ ),
dcc.Dropdown(
id="filter-models",
options=[{"label": m, "value": m} for m in models],
@@ -2304,7 +2440,9 @@ def build_filter_bar(df):
),
dbc.Col(
[
- html.Label("Root Notes", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}),
+ html.Label(
+ "Root Notes", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}
+ ),
dcc.Dropdown(
id="filter-roots",
options=[{"label": r, "value": r} for r in roots],
@@ -2317,7 +2455,9 @@ def build_filter_bar(df):
),
dbc.Col(
[
- html.Label("Scale Type", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}),
+ html.Label(
+ "Scale Type", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}
+ ),
dcc.Dropdown(
id="filter-scales",
options=[{"label": s.title(), "value": s} for s in scales],
@@ -2330,11 +2470,14 @@ def build_filter_bar(df):
),
dbc.Col(
[
- html.Label("Variation", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}),
+ html.Label(
+ "Variation", style={"color": PLOTLY_TEXT, "fontSize": "0.8rem"}
+ ),
dcc.Dropdown(
id="filter-variations",
options=[
- {"label": v.replace("_", " ").title(), "value": v} for v in variations
+ {"label": v.replace("_", " ").title(), "value": v}
+ for v in variations
],
value=variations,
multi=True,
@@ -2465,7 +2608,9 @@ def create_app(run_path):
app.layout = dbc.Container(
[
# Hidden store for the full data
- dcc.Store(id="run-data", data=df.to_json(date_format="iso", orient="split")),
+ dcc.Store(
+ id="run-data", data=df.to_json(date_format="iso", orient="split")
+ ),
# Header
dbc.Row(
[
@@ -2512,7 +2657,9 @@ def create_app(run_path):
def update_overview(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
total = len(filtered)
eligible = _eligible_overall_rows(filtered)
@@ -2539,7 +2686,11 @@ def update_overview(models, roots, scales, variations):
avg_latency = filtered["api_latency"].mean()
# Best / worst model
- model_rates = eligible.groupby("model")["overall_pass"].mean().sort_values(ascending=False)
+ model_rates = (
+ eligible.groupby("model")["overall_pass"]
+ .mean()
+ .sort_values(ascending=False)
+ )
best_model = (
f"{model_rates.index[0]} ({model_rates.iloc[0] * 100:.1f}%)"
if len(model_rates) > 0
@@ -2556,7 +2707,9 @@ def update_overview(models, roots, scales, variations):
# Metric cards row
dbc.Row(
[
- dbc.Col(make_metric_card("Total Generations", str(total)), md=2),
+ dbc.Col(
+ make_metric_card("Total Generations", str(total)), md=2
+ ),
dbc.Col(
make_metric_card(
"Pass Rate",
@@ -2571,7 +2724,9 @@ def update_overview(models, roots, scales, variations):
md=2,
),
dbc.Col(
- make_metric_card("Worst Model", worst_model, color="#e74c3c"),
+ make_metric_card(
+ "Worst Model", worst_model, color="#e74c3c"
+ ),
md=2,
),
dbc.Col(
@@ -2585,7 +2740,9 @@ def update_overview(models, roots, scales, variations):
dbc.Col(
make_metric_card(
"Avg Successful Latency",
- f"{avg_latency:.1f}s" if pd.notna(avg_latency) else "N/A",
+ f"{avg_latency:.1f}s"
+ if pd.notna(avg_latency)
+ else "N/A",
),
md=2,
),
@@ -2595,7 +2752,9 @@ def update_overview(models, roots, scales, variations):
# Main chart
dbc.Row(
[
- dbc.Col(dcc.Graph(figure=build_pass_rate_by_model(filtered)), md=12),
+ dbc.Col(
+ dcc.Graph(figure=build_pass_rate_by_model(filtered)), md=12
+ ),
]
),
]
@@ -2613,14 +2772,18 @@ def update_overview(models, roots, scales, variations):
def update_model(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
return html.Div(
[
dbc.Row(
[
dbc.Col(
- dcc.Graph(figure=build_duration_adherence_by_model(filtered)),
+ dcc.Graph(
+ figure=build_duration_adherence_by_model(filtered)
+ ),
md=6,
),
dbc.Col(
@@ -2634,11 +2797,15 @@ def update_model(models, roots, scales, variations):
dbc.Row(
[
dbc.Col(
- dcc.Graph(figure=build_texture_performance_by_model(filtered)),
+ dcc.Graph(
+ figure=build_texture_performance_by_model(filtered)
+ ),
md=6,
),
dbc.Col(
- dcc.Graph(figure=build_chord_performance_by_model(filtered)),
+ dcc.Graph(
+ figure=build_chord_performance_by_model(filtered)
+ ),
md=6,
),
],
@@ -2646,7 +2813,9 @@ def update_model(models, roots, scales, variations):
),
dbc.Row(
[
- dbc.Col(dcc.Graph(figure=build_model_root_heatmap(filtered)), md=12),
+ dbc.Col(
+ dcc.Graph(figure=build_model_root_heatmap(filtered)), md=12
+ ),
]
),
]
@@ -2664,26 +2833,32 @@ def update_model(models, roots, scales, variations):
def update_root_scale(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
return html.Div(
[
dbc.Row(
[
dbc.Col(dcc.Graph(figure=build_root_pass_rate(filtered)), md=6),
- dbc.Col(dcc.Graph(figure=build_root_scale_grouped(filtered)), md=6),
+ dbc.Col(
+ dcc.Graph(figure=build_root_scale_grouped(filtered)), md=6
+ ),
],
className="mb-3",
),
dbc.Row(
[
- dbc.Col(dcc.Graph(figure=build_root_scale_heatmap(filtered)), md=12),
+ dbc.Col(
+ dcc.Graph(figure=build_root_scale_heatmap(filtered)), md=12
+ ),
]
),
]
)
- @app.callback( # noqa: E303
+ @app.callback(
Output("tab-latency-content", "children"),
[
Input("filter-models", "value"),
@@ -2695,7 +2870,9 @@ def update_root_scale(models, roots, scales, variations):
def update_latency(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
return html.Div(
[
@@ -2707,7 +2884,9 @@ def update_latency(models, roots, scales, variations):
),
dbc.Row(
[
- dbc.Col(dcc.Graph(figure=build_latency_vs_pass(filtered)), md=12),
+ dbc.Col(
+ dcc.Graph(figure=build_latency_vs_pass(filtered)), md=12
+ ),
]
),
]
@@ -2725,7 +2904,9 @@ def update_latency(models, roots, scales, variations):
def update_cost(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
return html.Div(
[
@@ -2753,7 +2934,9 @@ def update_cost(models, roots, scales, variations):
def update_reasoning(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
# Summary cards for reasoning impact
effort_rows = filtered[filtered["effort"].notna()]
@@ -2763,14 +2946,24 @@ def update_reasoning(models, roots, scales, variations):
[
bm
for bm in filtered["base_model"].unique()
- if not filtered.loc[filtered["base_model"] == bm, "use_thinking"].all()
- and filtered.loc[filtered["base_model"] == bm, "use_thinking"].any()
- and filtered.loc[filtered["base_model"] == bm, "effort"].isna().all()
+ if not filtered.loc[
+ filtered["base_model"] == bm, "use_thinking"
+ ].all()
+ and filtered.loc[
+ filtered["base_model"] == bm, "use_thinking"
+ ].any()
+ and filtered.loc[filtered["base_model"] == bm, "effort"]
+ .isna()
+ .all()
]
)
]
- n_effort_models = effort_rows["base_model"].nunique() if not effort_rows.empty else 0
- n_toggle_models = toggle_rows["base_model"].nunique() if not toggle_rows.empty else 0
+ n_effort_models = (
+ effort_rows["base_model"].nunique() if not effort_rows.empty else 0
+ )
+ n_toggle_models = (
+ toggle_rows["base_model"].nunique() if not toggle_rows.empty else 0
+ )
avg_pass_rate = _overall_pass_rate_percent(filtered)
cards = dbc.Row(
@@ -2816,7 +3009,9 @@ def update_reasoning(models, roots, scales, variations):
rows.append(
dbc.Row(
[
- dbc.Col(dcc.Graph(figure=build_effort_impact_delta(filtered)), md=12),
+ dbc.Col(
+ dcc.Graph(figure=build_effort_impact_delta(filtered)), md=12
+ ),
],
className="mb-3",
)
@@ -2827,7 +3022,9 @@ def update_reasoning(models, roots, scales, variations):
dbc.Row(
[
dbc.Col(
- dcc.Graph(figure=build_reasoning_toggle_comparison(filtered)),
+ dcc.Graph(
+ figure=build_reasoning_toggle_comparison(filtered)
+ ),
md=12,
),
],
@@ -2840,7 +3037,9 @@ def update_reasoning(models, roots, scales, variations):
dbc.Row(
[
dbc.Col(
- dcc.Graph(figure=build_reasoning_cost_effectiveness(filtered)),
+ dcc.Graph(
+ figure=build_reasoning_cost_effectiveness(filtered)
+ ),
md=12,
),
]
@@ -2861,7 +3060,9 @@ def update_reasoning(models, roots, scales, variations):
def update_errors(models, roots, scales, variations):
filtered = apply_filters(df, models, roots, scales, variations)
if filtered.empty:
- return html.P("No data matches current filters.", style={"color": PLOTLY_TEXT})
+ return html.P(
+ "No data matches current filters.", style={"color": PLOTLY_TEXT}
+ )
return html.Div(
[
@@ -2877,11 +3078,15 @@ def update_errors(models, roots, scales, variations):
dbc.Row(
[
dbc.Col(
- dcc.Graph(figure=build_incorrect_pitches_by_model(filtered)),
+ dcc.Graph(
+ figure=build_incorrect_pitches_by_model(filtered)
+ ),
md=6,
),
dbc.Col(
- dcc.Graph(figure=build_incorrect_intervals_by_model(filtered)),
+ dcc.Graph(
+ figure=build_incorrect_intervals_by_model(filtered)
+ ),
md=6,
),
],
@@ -2934,7 +3139,9 @@ def export_dashboard(n_clicks):
if has_reasoning:
figures["effort_impact_delta"] = build_effort_impact_delta(df)
figures["reasoning_toggle"] = build_reasoning_toggle_comparison(df)
- figures["reasoning_cost_effectiveness"] = build_reasoning_cost_effectiveness(df)
+ figures["reasoning_cost_effectiveness"] = (
+ build_reasoning_cost_effectiveness(df)
+ )
# Save individual charts
for name, fig in figures.items():
@@ -2943,7 +3150,7 @@ def export_dashboard(n_clicks):
# Build combined dashboard HTML
combined_html = _build_combined_html(figures, run_name, timestamp, totals, df)
combined_path = export_dir / "dashboard.html"
- with open(combined_path, "w", encoding="utf-8") as f:
+ with combined_path.open("w", encoding="utf-8") as f:
f.write(combined_html)
return dbc.Alert(
@@ -2998,8 +3205,12 @@ def _build_combined_html(figures, run_name, timestamp, totals, df):
chart_divs = []
for name, fig in figures.items():
title = name.replace("_", " ").title()
- div_html = fig.to_html(include_plotlyjs=False, full_html=False, div_id=f"chart-{name}")
- chart_divs.append(f'{title}
{div_html}')
+ div_html = fig.to_html(
+ include_plotlyjs=False, full_html=False, div_id=f"chart-{name}"
+ )
+ chart_divs.append(
+ f'{title}
{div_html}'
+ )
return f"""
@@ -3035,12 +3246,12 @@ def _build_combined_html(figures, run_name, timestamp, totals, df):
def main():
"""Entry point. Parses CLI args or prompts for run selection, then launches the dashboard."""
if len(sys.argv) > 1:
- run_path = sys.argv[1]
+ run_path = Path(sys.argv[1])
# Support both full path and just run name
- if not os.path.isdir(run_path):
+ if not run_path.is_dir():
# Try the evaluator's default output directory.
candidate = get_evaluations_dir() / run_path
- if os.path.isdir(candidate):
+ if candidate.is_dir():
run_path = candidate
else:
print(f"Run directory not found: {run_path}")
diff --git a/src/conductor_eval/checks.py b/src/conductor_eval/checks.py
index f9010c9..0634b5d 100644
--- a/src/conductor_eval/checks.py
+++ b/src/conductor_eval/checks.py
@@ -38,13 +38,15 @@ def scale_test(midi, root, scale):
# Validate root and scale.
try:
root_pc = note_name_to_pitch_class(root)
- except ValueError:
- raise ValueError(f"Invalid root note: {root}")
+ except ValueError as exc:
+ raise ValueError(f"Invalid root note: {root}") from exc
if scale.lower() not in SCALE_INTERVALS:
raise ValueError(f"Invalid scale mode: {scale.lower()}")
# Determine the acceptable pitch classes for the given scale.
- acceptable_pcs = [(root_pc + interval) % 12 for interval in SCALE_INTERVALS[scale.lower()]]
+ acceptable_pcs = [
+ (root_pc + interval) % 12 for interval in SCALE_INTERVALS[scale.lower()]
+ ]
# print(f"Root Note: {root}, Scale Mode: {scale}, Acceptable Pitch Classes: {acceptable_pcs}")
correct = 0
@@ -65,7 +67,7 @@ def scale_test(midi, root, scale):
incorrect += 1
incorrect_pitches.add(pitch_class)
- results = {
+ return {
"total": total,
"correct": correct,
"incorrect": incorrect,
@@ -74,7 +76,6 @@ def scale_test(midi, root, scale):
"incorrect": list(incorrect_pitches),
},
}
- return results
def duration_test(midi, duration):
@@ -92,7 +93,9 @@ def duration_test(midi, duration):
raise ValueError(f"Invalid duration: {duration}")
ticks_per_beat = midi.ticks_per_beat
- expected_ticks = beats_to_ticks(DURATION_BEATS[duration], ticks_per_beat, "duration")
+ expected_ticks = beats_to_ticks(
+ DURATION_BEATS[duration], ticks_per_beat, "duration"
+ )
# print(f"Expected duration in ticks: {expected_ticks}")
total = 0
@@ -108,13 +111,12 @@ def duration_test(midi, duration):
incorrect_lengths[ratio] = incorrect_lengths.get(ratio, 0) + 1
else:
correct += 1
- results = {
+ return {
"total": total,
"correct": correct,
"incorrect": incorrect,
"lengths": incorrect_lengths,
}
- return results
def monophony_test(midi):
@@ -129,7 +131,11 @@ def monophony_test(midi):
def polyphony_test(midi, min_voices=2):
"""Test whether the MIDI reaches a requested number of simultaneous voices."""
- if not isinstance(min_voices, int) or isinstance(min_voices, bool) or min_voices < 2:
+ if (
+ not isinstance(min_voices, int)
+ or isinstance(min_voices, bool)
+ or min_voices < 2
+ ):
raise ValueError("min_voices must be an integer greater than or equal to 2")
profile = calculate_polyphony_profile(midi)
@@ -159,7 +165,9 @@ def _resolve_diatonic_triads(root, scale, progression):
for numeral in progression:
if not isinstance(numeral, str) or numeral.upper() not in _ROMAN_DEGREES:
supported = ", ".join(_ROMAN_DEGREES)
- raise ValueError(f"Unsupported Roman numeral {numeral!r}; expected one of: {supported}")
+ raise ValueError(
+ f"Unsupported Roman numeral {numeral!r}; expected one of: {supported}"
+ )
degree = _ROMAN_DEGREES[numeral.upper()]
pitch_classes = {
scale_pcs[degree],
@@ -185,7 +193,9 @@ def chord_progression_test(
"""
if not isinstance(strict, bool):
raise ValueError("strict must be a boolean")
- chord_ticks = beats_to_ticks(beats_per_chord, midi.ticks_per_beat, "beats_per_chord")
+ chord_ticks = beats_to_ticks(
+ beats_per_chord, midi.ticks_per_beat, "beats_per_chord"
+ )
if chord_ticks == 0:
raise ValueError("beats_per_chord must be greater than zero")
@@ -196,7 +206,9 @@ def chord_progression_test(
for index, (numeral, expected_pcs) in enumerate(expected_chords):
onset_tick = index * chord_ticks
actual_pcs = {
- note.pitch % 12 for note in intervals if note.start_tick <= onset_tick < note.end_tick
+ note.pitch % 12
+ for note in intervals
+ if note.start_tick <= onset_tick < note.end_tick
}
missing = expected_pcs - actual_pcs
extra = actual_pcs - expected_pcs
@@ -226,7 +238,8 @@ def harmonic_rhythm_test(midi, expected_onsets):
if not isinstance(expected_onsets, list) or not expected_onsets:
raise ValueError("expected_onsets must be a non-empty list")
expected_ticks = {
- beats_to_ticks(beat, midi.ticks_per_beat, "expected_onsets") for beat in expected_onsets
+ beats_to_ticks(beat, midi.ticks_per_beat, "expected_onsets")
+ for beat in expected_onsets
}
if len(expected_ticks) != len(expected_onsets):
raise ValueError("expected_onsets must contain unique beat positions")
@@ -252,10 +265,12 @@ def chord_event_positions_test(midi, expected_starts, expected_ends):
if not isinstance(expected_starts, list) or not isinstance(expected_ends, list):
raise ValueError("expected_starts and expected_ends must be lists")
if not expected_starts or len(expected_starts) != len(expected_ends):
- raise ValueError("expected_starts and expected_ends must have the same non-zero length")
+ raise ValueError(
+ "expected_starts and expected_ends must have the same non-zero length"
+ )
expected_pairs = set()
- for start, end in zip(expected_starts, expected_ends):
+ for start, end in zip(expected_starts, expected_ends, strict=True):
start_tick = beats_to_ticks(start, midi.ticks_per_beat, "expected_starts")
end_tick = beats_to_ticks(end, midi.ticks_per_beat, "expected_ends")
if end_tick <= start_tick:
diff --git a/src/conductor_eval/evaluator.py b/src/conductor_eval/evaluator.py
index 0fdfe71..45b6d85 100644
--- a/src/conductor_eval/evaluator.py
+++ b/src/conductor_eval/evaluator.py
@@ -14,9 +14,9 @@
import logging
import time
import traceback
-from datetime import datetime
+from datetime import datetime, timezone
from pathlib import Path
-from typing import Union
+from typing import ClassVar
from uuid import uuid4
from conductor_core import EngineConfig, GenerationRequest, LoopGenerationEngine
@@ -132,10 +132,10 @@ class Evaluator:
AVAILABLE_TESTS: Registry of available test functions
"""
- SCALES = ["major", "minor"]
+ SCALES: ClassVar[list[str]] = ["major", "minor"]
MAX_CLOUD_CONCURRENCY = 4
- AVAILABLE_TESTS = {
+ AVAILABLE_TESTS: ClassVar[dict[str, object]] = {
"scale": scale_test,
"duration": duration_test,
"monophony": monophony_test,
@@ -190,13 +190,17 @@ def __init__(
the ``evaluations`` subdirectory in Eval's data directory.
temperature: Default temperature for generation.
"""
- self.output_dir = get_evaluations_dir() if output_dir is None else Path(output_dir)
+ self.output_dir = (
+ get_evaluations_dir() if output_dir is None else Path(output_dir)
+ )
self.temperature = temperature
self.console = Console(force_terminal=True)
self.model_info = get_model_info()
@staticmethod
- def _create_run_logger(run_path: Path, run_id: str) -> tuple[logging.Logger, logging.Handler]:
+ def _create_run_logger(
+ run_path: Path, run_id: str
+ ) -> tuple[logging.Logger, logging.Handler]:
"""Create an isolated file logger for one evaluation run."""
logger = logging.Logger(f"{__name__}.run.{run_id}")
logger.setLevel(logging.INFO)
@@ -226,11 +230,11 @@ def _format_traceback(error: Exception) -> str:
def evaluate(
self,
- prompts: Union[str, list[str]],
+ prompts: str | list[str],
roots: list[str],
- models: Union[str, list[str]] = "all",
- run_name: str = None,
- tests: list[str] = ["scale", "duration"],
+ models: str | list[str] = "all",
+ run_name: str | None = None,
+ tests: list[str] | None = None,
test_reasoning: bool = False,
test_params: dict[str, dict] | None = None,
max_cloud_concurrency: int = MAX_CLOUD_CONCURRENCY,
@@ -256,6 +260,8 @@ def evaluate(
Raises:
ValueError: If run_name is not provided.
"""
+ if tests is None:
+ tests = ["scale", "duration"]
if run_name is None:
raise ValueError("run_name is required")
if (
@@ -294,7 +300,7 @@ def evaluate(
"temperature": self.temperature,
"max_cloud_concurrency": max_cloud_concurrency,
}
- with open(run_path / "config.json", "w", encoding="utf-8") as f:
+ with (run_path / "config.json").open("w", encoding="utf-8") as f:
json.dump(config, f, indent=2)
# Generate all task combinations
@@ -306,11 +312,15 @@ def evaluate(
test_reasoning=test_reasoning,
test_params=test_params,
)
- logger.info("Starting evaluation '%s' with %d total tasks", run_name, len(tasks))
+ logger.info(
+ "Starting evaluation '%s' with %d total tasks", run_name, len(tasks)
+ )
# Separate async and sync tasks
async_tasks = [t for t in tasks if self._is_async_provider(t["provider"])]
- sync_tasks = [t for t in tasks if not self._is_async_provider(t["provider"])]
+ sync_tasks = [
+ t for t in tasks if not self._is_async_provider(t["provider"])
+ ]
all_results = []
@@ -334,7 +344,7 @@ def evaluate(
# Generate and save summary
summary = self._generate_summary(all_results, config)
- with open(run_path / "summary.json", "w", encoding="utf-8") as f:
+ with (run_path / "summary.json").open("w", encoding="utf-8") as f:
json.dump(summary, f, indent=2)
logger.info("Evaluation complete. Results saved to %s", run_path)
@@ -397,7 +407,9 @@ def run_tests(
try:
test_result = test_func(midi_data, root, scale)
test_result["ran"] = True
- test_result["eligible"] = self._has_substantive_evidence(test_name, test_result)
+ test_result["eligible"] = self._has_substantive_evidence(
+ test_name, test_result
+ )
test_result["passed"] = (
test_result["eligible"] and test_result.get("incorrect", 0) == 0
)
@@ -444,7 +456,8 @@ def run_tests(
test_name, test_result
)
test_result["passed"] = (
- test_result["eligible"] and test_result.get("incorrect", 0) == 0
+ test_result["eligible"]
+ and test_result.get("incorrect", 0) == 0
)
test_result["status"] = (
"passed"
@@ -454,7 +467,9 @@ def run_tests(
else "ineligible"
)
test_result["params"] = {"duration": duration_value}
- test_result["detected_from_prompt"] = "duration" not in explicit_params
+ test_result["detected_from_prompt"] = (
+ "duration" not in explicit_params
+ )
results[test_name] = test_result
if test_result["eligible"]:
substantive_checks += 1
@@ -478,7 +493,9 @@ def run_tests(
try:
test_result = test_func(midi_data, **resolved_params)
test_result["ran"] = True
- test_result["eligible"] = self._has_substantive_evidence(test_name, test_result)
+ test_result["eligible"] = self._has_substantive_evidence(
+ test_name, test_result
+ )
test_result["passed"] = test_result["eligible"] and test_result.get(
"passed", test_result.get("incorrect", 0) == 0
)
@@ -518,7 +535,9 @@ def run_tests(
return results
@staticmethod
- def _validate_test_params(tests: list[str], test_params: dict[str, dict] | None) -> dict:
+ def _validate_test_params(
+ tests: list[str], test_params: dict[str, dict] | None
+ ) -> dict:
"""Validate and copy explicit test arguments."""
if test_params is None:
return {}
@@ -527,7 +546,9 @@ def _validate_test_params(tests: list[str], test_params: dict[str, dict] | None)
unselected = sorted(set(test_params) - set(tests))
if unselected:
- raise ValueError("test_params contains unselected tests: " + ", ".join(unselected))
+ raise ValueError(
+ "test_params contains unselected tests: " + ", ".join(unselected)
+ )
validated = {}
for test_name, params in test_params.items():
@@ -537,7 +558,7 @@ def _validate_test_params(tests: list[str], test_params: dict[str, dict] | None)
return validated
def _resolve_models(
- self, models: Union[str, list[str]], logger: logging.Logger
+ self, models: str | list[str], logger: logging.Logger
) -> list[tuple[str, str]]:
"""
Resolve model specification to (provider, model_name) tuples.
@@ -556,25 +577,35 @@ def _resolve_models(
# All cloud models from model_list.json
for provider in ["OpenAI", "Anthropic", "Google"]:
if provider in self.model_info["models"]:
- for model in self.model_info["models"][provider].keys():
- resolved.append((provider, model))
+ resolved.extend(
+ (provider, model)
+ for model in self.model_info["models"][provider]
+ )
# All Ollama models
- resolved.extend(("Ollama", model) for model in self._discover_ollama_models())
+ resolved.extend(
+ ("Ollama", model) for model in self._discover_ollama_models()
+ )
elif models_lower == "openai":
- for model in self.model_info["models"]["OpenAI"].keys():
- resolved.append(("OpenAI", model))
+ resolved.extend(
+ ("OpenAI", model) for model in self.model_info["models"]["OpenAI"]
+ )
elif models_lower == "anthropic":
- for model in self.model_info["models"]["Anthropic"].keys():
- resolved.append(("Anthropic", model))
+ resolved.extend(
+ ("Anthropic", model)
+ for model in self.model_info["models"]["Anthropic"]
+ )
elif models_lower == "google":
- for model in self.model_info["models"]["Google"].keys():
- resolved.append(("Google", model))
+ resolved.extend(
+ ("Google", model) for model in self.model_info["models"]["Google"]
+ )
elif models_lower == "ollama":
- resolved.extend(("Ollama", model) for model in self._discover_ollama_models())
+ resolved.extend(
+ ("Ollama", model) for model in self._discover_ollama_models()
+ )
else:
# Assume it's a single model name
@@ -612,9 +643,11 @@ def _get_provider(self, model: str) -> str:
"""
# Check cloud providers first
for provider in ["OpenAI", "Anthropic", "Google"]:
- if provider in self.model_info["models"]:
- if model in self.model_info["models"][provider]:
- return provider
+ if (
+ provider in self.model_info["models"]
+ and model in self.model_info["models"][provider]
+ ):
+ return provider
if model in self._discover_ollama_models():
return "Ollama"
@@ -647,9 +680,11 @@ def _get_model_capabilities(self, provider: str, model: str) -> dict:
if provider == "Ollama":
return {"extended_thinking": False, "effort_options": []}
- if provider in self.model_info["models"]:
- if model in self.model_info["models"][provider]:
- return self.model_info["models"][provider][model]
+ if (
+ provider in self.model_info["models"]
+ and model in self.model_info["models"][provider]
+ ):
+ return self.model_info["models"][provider][model]
return {"extended_thinking": False, "effort_options": []}
@@ -710,24 +745,24 @@ def _generate_tasks(
test_reasoning=test_reasoning,
)
- for variation in variations:
- tasks.append(
- {
- "provider": provider,
- "model": model,
- "original_prompt": prompt,
- "full_prompt": full_prompt,
- "root": root,
- "scale": scale,
- "use_thinking": variation["use_thinking"],
- "effort": variation["effort"],
- "variation_name": variation["name"],
- "test_params": {
- name: dict(params)
- for name, params in (test_params or {}).items()
- },
- }
- )
+ tasks.extend(
+ {
+ "provider": provider,
+ "model": model,
+ "original_prompt": prompt,
+ "full_prompt": full_prompt,
+ "root": root,
+ "scale": scale,
+ "use_thinking": variation["use_thinking"],
+ "effort": variation["effort"],
+ "variation_name": variation["name"],
+ "test_params": {
+ name: dict(params)
+ for name, params in (test_params or {}).items()
+ },
+ }
+ for variation in variations
+ )
occurrences: dict[str, int] = {}
for task in tasks:
@@ -741,7 +776,9 @@ def _generate_tasks(
return tasks
- def _generate_variations(self, model: str, provider: str, test_reasoning: bool) -> list[dict]:
+ def _generate_variations(
+ self, model: str, provider: str, test_reasoning: bool
+ ) -> list[dict]:
"""
Generate all config variations to test for a model.
@@ -760,25 +797,17 @@ def _generate_variations(self, model: str, provider: str, test_reasoning: bool)
if test_reasoning and supports_thinking:
# For OpenAI reasoning models (o-series), only effort levels matter.
- if provider == "OpenAI" and supports_thinking:
- for effort in effort_options:
- variations.append(
- {
- "use_thinking": True,
- "effort": effort,
- "name": effort,
- }
- )
- # For Anthropic/Google, test thinking with effort levels when supported.
- elif provider in ["Anthropic", "Google"] and effort_options:
- for effort in effort_options:
- variations.append(
- {
- "use_thinking": True,
- "effort": effort,
- "name": effort,
- }
- )
+ if (provider == "OpenAI" and supports_thinking) or (
+ provider in ["Anthropic", "Google"] and effort_options
+ ):
+ variations.extend(
+ {
+ "use_thinking": True,
+ "effort": effort,
+ "name": effort,
+ }
+ for effort in effort_options
+ )
# For Anthropic/Google with a reasoning toggle but no effort options.
elif provider in ["Anthropic", "Google"]:
variations.append(
@@ -805,15 +834,9 @@ def _generate_variations(self, model: str, provider: str, test_reasoning: bool)
)
else:
# No reasoning testing: use the default effort for effort-based models.
- if supports_thinking and provider == "OpenAI":
- variations.append(
- {
- "use_thinking": True,
- "effort": effort_options[0],
- "name": effort_options[0],
- }
- )
- elif effort_options and provider in ["Anthropic", "Google"]:
+ if (supports_thinking and provider == "OpenAI") or (
+ effort_options and provider in ["Anthropic", "Google"]
+ ):
variations.append(
{
"use_thinking": True,
@@ -852,7 +875,9 @@ async def _run_async_batch(
list: List of result dictionaries
"""
providers = {task["provider"] for task in tasks}
- semaphores = {provider: asyncio.Semaphore(max_cloud_concurrency) for provider in providers}
+ semaphores = {
+ provider: asyncio.Semaphore(max_cloud_concurrency) for provider in providers
+ }
results = []
total_tasks = len(tasks)
@@ -888,7 +913,9 @@ async def run_single_task(task: dict) -> dict:
results.append(result)
# Update table
- new_table = Table(title=f"Evaluation Progress ({len(results)}/{total_tasks})")
+ new_table = Table(
+ title=f"Evaluation Progress ({len(results)}/{total_tasks})"
+ )
new_table.add_column("Provider")
new_table.add_column("Model")
new_table.add_column("Eligible")
@@ -926,11 +953,17 @@ async def run_single_task(task: dict) -> dict:
s["cost_count"] += 1
for (provider, model), s in stats.items():
- pass_rate = s["passed"] / s["eligible"] * 100 if s["eligible"] else 0
+ pass_rate = (
+ s["passed"] / s["eligible"] * 100 if s["eligible"] else 0
+ )
avg_latency = (
- s["latency_sum"] / s["latency_count"] if s["latency_count"] else None
+ s["latency_sum"] / s["latency_count"]
+ if s["latency_count"]
+ else None
+ )
+ avg_cost = (
+ s["cost_sum"] / s["cost_count"] if s["cost_count"] else None
)
- avg_cost = s["cost_sum"] / s["cost_count"] if s["cost_count"] else None
new_table.add_row(
provider,
model,
@@ -975,12 +1008,14 @@ def _run_sync_batch(
table.add_column("Avg Latency")
with Live(table, console=self.console, refresh_per_second=2) as live:
- for i, task in enumerate(tasks):
+ for task in tasks:
result = self._run_single(task, run_path, tests_to_run, logger)
results.append(result)
# Update table
- new_table = Table(title=f"Evaluation Progress ({len(results)}/{total_tasks})")
+ new_table = Table(
+ title=f"Evaluation Progress ({len(results)}/{total_tasks})"
+ )
new_table.add_column("Model")
new_table.add_column("Eligible")
new_table.add_column("Pass Rate")
@@ -1010,9 +1045,13 @@ def _run_sync_batch(
s["latency_count"] += 1
for model, s in stats.items():
- pass_rate = s["passed"] / s["eligible"] * 100 if s["eligible"] else 0
+ pass_rate = (
+ s["passed"] / s["eligible"] * 100 if s["eligible"] else 0
+ )
avg_latency = (
- s["latency_sum"] / s["latency_count"] if s["latency_count"] else None
+ s["latency_sum"] / s["latency_count"]
+ if s["latency_count"]
+ else None
)
new_table.add_row(
model,
@@ -1114,7 +1153,9 @@ def _run_single(
result["error"] = str(e)
result["tests"]["overall_pass"] = False
result["tests"]["overall_status"] = (
- "rate_limited" if isinstance(e, ProviderRateLimitError) else "generation_error"
+ "rate_limited"
+ if isinstance(e, ProviderRateLimitError)
+ else "generation_error"
)
# Still save the result even on failure
self._save_results(result, None, [], run_path, task)
@@ -1164,12 +1205,12 @@ def _save_results(
# Save messages (for fine-tuning)
messages_path = result_dir / "messages.json"
- with open(messages_path, "w", encoding="utf-8") as f:
+ with messages_path.open("w", encoding="utf-8") as f:
json.dump(messages, f, indent=2)
# Save test results
results_path = result_dir / "test_results.json"
- with open(results_path, "w", encoding="utf-8") as f:
+ with results_path.open("w", encoding="utf-8") as f:
json.dump(result, f, indent=2)
def _generate_summary(self, all_results: list[dict], config: dict) -> dict:
@@ -1368,7 +1409,7 @@ def _generate_summary(self, all_results: list[dict], config: dict) -> dict:
summary["totals"]["overall_pass_count"] / eligible_total
)
- for model, m in summary["by_model"].items():
+ for m in summary["by_model"].values():
if m["eligible"] > 0:
m["pass_rate"] = m["passed"] / m["eligible"]
if m["successful_latency_count"]:
@@ -1380,11 +1421,11 @@ def _generate_summary(self, all_results: list[dict], config: dict) -> dict:
/ summary["totals"]["successful_latency_count"]
)
- for root, r in summary["by_root"].items():
+ for r in summary["by_root"].values():
if r["eligible"] > 0:
r["pass_rate"] = r["passed"] / r["eligible"]
- for scale, s in summary["by_scale"].items():
+ for s in summary["by_scale"].values():
if s["eligible"] > 0:
s["pass_rate"] = s["passed"] / s["eligible"]
@@ -1414,7 +1455,7 @@ def _sanitize_filename(self, text: str, max_len: int = 50) -> str:
def _create_run_directory(self, run_name: str) -> tuple[Path, str, str]:
"""Create a collision-resistant directory and return its authoritative metadata."""
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+ timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
run_id = f"{timestamp}_{self._sanitize_filename(run_name, max_len=32)}_{uuid4().hex[:16]}"
run_path = self.output_dir / run_id
run_path.mkdir(parents=True, exist_ok=False)
@@ -1438,7 +1479,9 @@ def _task_fingerprint(task: dict) -> str:
"test_params",
)
}
- canonical = json.dumps(task_inputs, sort_keys=True, separators=(",", ":"), default=str)
+ canonical = json.dumps(
+ task_inputs, sort_keys=True, separators=(",", ":"), default=str
+ )
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
@@ -1448,8 +1491,8 @@ def main() -> None:
if not confirm_direct_evaluation():
raise SystemExit(1)
- eval = Evaluator(output_dir="runs", temperature=0.0)
- eval.evaluate(
+ evaluator = Evaluator(output_dir="runs", temperature=0.0)
+ evaluator.evaluate(
prompts=[
"An arpeggiator in only quarter notes",
"An arpeggiator in only eighth notes",
diff --git a/src/conductor_eval/midi.py b/src/conductor_eval/midi.py
index 6e5c255..8c6f8cc 100644
--- a/src/conductor_eval/midi.py
+++ b/src/conductor_eval/midi.py
@@ -70,8 +70,12 @@ def extract_note_intervals(midi: MidiFile) -> list[NoteInterval]:
absolute_tick += msg.time
if msg.type == "note_on" and msg.velocity > 0:
key = (msg.channel, msg.note)
- active_notes.setdefault(key, deque()).append((absolute_tick, msg.velocity))
- elif msg.type == "note_off" or (msg.type == "note_on" and msg.velocity == 0):
+ active_notes.setdefault(key, deque()).append(
+ (absolute_tick, msg.velocity)
+ )
+ elif msg.type == "note_off" or (
+ msg.type == "note_on" and msg.velocity == 0
+ ):
key = (msg.channel, msg.note)
starts = active_notes.get(key)
if not starts:
@@ -119,7 +123,9 @@ def calculate_polyphony_profile(midi: MidiFile) -> dict:
for tick in sorted(events):
elapsed = tick - previous_tick
if elapsed:
- distribution_ticks[active_notes] = distribution_ticks.get(active_notes, 0) + elapsed
+ distribution_ticks[active_notes] = (
+ distribution_ticks.get(active_notes, 0) + elapsed
+ )
active_notes += events[tick]
max_polyphony = max(max_polyphony, active_notes)
previous_tick = tick
@@ -139,6 +145,8 @@ def calculate_polyphony_profile(midi: MidiFile) -> dict:
"polyphony_distribution": distribution,
"polyphony_percentages": percentages,
"max_polyphony": max_polyphony,
- "total_duration": round(ticks_to_beats(total_duration_ticks, midi.ticks_per_beat), 4),
+ "total_duration": round(
+ ticks_to_beats(total_duration_ticks, midi.ticks_per_beat), 4
+ ),
"ticks_per_beat": midi.ticks_per_beat,
}
diff --git a/src/conductor_eval/paths.py b/src/conductor_eval/paths.py
index 5ad930d..d9a89ac 100644
--- a/src/conductor_eval/paths.py
+++ b/src/conductor_eval/paths.py
@@ -1,8 +1,8 @@
"""Resolve Conductor Eval's mutable data directories."""
import os
+from collections.abc import Mapping
from pathlib import Path
-from typing import Mapping
PROJECT_ID = "eval"
PROJECT_DATA_ENV = "CONDUCTOR_EVAL_HOME"
@@ -12,7 +12,9 @@
PROJECT_HOME_ENV = PROJECT_DATA_ENV
-def _environment_path(name: str, environ: Mapping[str, str] | None = None) -> Path | None:
+def _environment_path(
+ name: str, environ: Mapping[str, str] | None = None
+) -> Path | None:
"""Return an expanded environment path when the variable is set."""
env = os.environ if environ is None else environ
value = env.get(name)
diff --git a/tests/test_analysis.py b/tests/test_analysis.py
index dff5160..f646e88 100644
--- a/tests/test_analysis.py
+++ b/tests/test_analysis.py
@@ -56,8 +56,18 @@ def test_format_pass_rate_summary_uses_compact_outcome_text():
def test_metric_charts_exclude_unknown_costs_and_failed_latencies():
df = pd.DataFrame(
[
- {"model": "reported", "cost": 0.5, "api_latency": 2.0, "overall_pass": True},
- {"model": "unknown", "cost": None, "api_latency": None, "overall_pass": False},
+ {
+ "model": "reported",
+ "cost": 0.5,
+ "api_latency": 2.0,
+ "overall_pass": True,
+ },
+ {
+ "model": "unknown",
+ "cost": None,
+ "api_latency": None,
+ "overall_pass": False,
+ },
]
)
@@ -78,7 +88,9 @@ def test_latency_vs_pass_uses_all_attempts_for_pass_rate():
trace = build_latency_vs_pass(df).data[0]
stats = {
model: (latency, pass_rate)
- for model, latency, pass_rate in zip(trace.customdata, trace.x, trace.y, strict=True)
+ for model, latency, pass_rate in zip(
+ trace.customdata, trace.x, trace.y, strict=True
+ )
}
assert stats == {"alpha": (2.0, 50.0), "beta": (4.0, 0.0)}
@@ -273,8 +285,12 @@ def test_load_run_flattens_new_check_results(tmp_path):
assert [bar["bar"] for bar in row["chord_progression_failed_bars"]] == [2]
assert row["harmonic_rhythm_missing_onsets"] == [8.0]
assert row["harmonic_rhythm_unexpected_onsets"] == [6.0]
- assert row["chord_event_positions_missing"] == [{"start_beat": 8.0, "end_beat": 12.0}]
- assert row["chord_event_positions_unexpected"] == [{"start_beat": 8.0, "end_beat": 10.0}]
+ assert row["chord_event_positions_missing"] == [
+ {"start_beat": 8.0, "end_beat": 12.0}
+ ]
+ assert row["chord_event_positions_unexpected"] == [
+ {"start_beat": 8.0, "end_beat": 10.0}
+ ]
def test_load_run_uses_persisted_metadata_with_arbitrary_task_directory_names(tmp_path):
@@ -374,7 +390,10 @@ def test_load_run_excludes_ineligible_results_from_overall_pass_rates(tmp_path):
{
"overall_status": "ineligible",
"scale": {"ran": True, "total": 0, "correct": 0, "incorrect": 0},
- "duration": {"ran": False, "skipped": "No duration keyword detected in prompt"},
+ "duration": {
+ "ran": False,
+ "skipped": "No duration keyword detected in prompt",
+ },
},
)
@@ -648,7 +667,9 @@ def test_tradeoff_charts_compact_long_labels_and_keep_full_names_in_hover():
trace = figure.data[0]
long_model_index = list(trace.customdata).index(long_model)
compact_annotation = next(
- annotation for annotation in figure.layout.annotations if "…" in annotation.text
+ annotation
+ for annotation in figure.layout.annotations
+ if "…" in annotation.text
)
assert trace.mode == "markers"
@@ -770,7 +791,10 @@ def test_model_variant_order_has_deterministic_fallbacks():
"alpha (none)",
]
df = pd.DataFrame(
- [{"model": model, "api_latency": float(index)} for index, model in enumerate(models)]
+ [
+ {"model": model, "api_latency": float(index)}
+ for index, model in enumerate(models)
+ ]
)
assert [trace.name for trace in build_latency_box(df).data] == [
diff --git a/tests/test_checks.py b/tests/test_checks.py
index 330aa34..75ad8b8 100644
--- a/tests/test_checks.py
+++ b/tests/test_checks.py
@@ -51,7 +51,11 @@ def make_timed_midi(notes):
def block_chord_notes(chords):
- return [(pitch, bar * 4, (bar + 1) * 4) for bar, chord in enumerate(chords) for pitch in chord]
+ return [
+ (pitch, bar * 4, (bar + 1) * 4)
+ for bar, chord in enumerate(chords)
+ for pitch in chord
+ ]
def run_harmonic_checks(midi, root, scale):
diff --git a/tests/test_conductor_eval_direct_run_guard.py b/tests/test_conductor_eval_direct_run_guard.py
index 5010273..010fb7c 100644
--- a/tests/test_conductor_eval_direct_run_guard.py
+++ b/tests/test_conductor_eval_direct_run_guard.py
@@ -20,7 +20,10 @@ def test_direct_evaluator_run_aborts_before_creating_outputs(monkeypatch, tmp_pa
def test_direct_evaluation_confirmation_requires_exact_phrase():
assert confirm_direct_evaluation(lambda _prompt: "y") is False
- assert confirm_direct_evaluation(lambda _prompt: DIRECT_EVALUATION_CONFIRMATION) is True
+ assert (
+ confirm_direct_evaluation(lambda _prompt: DIRECT_EVALUATION_CONFIRMATION)
+ is True
+ )
def test_direct_evaluation_confirmation_handles_closed_stdin():
diff --git a/tests/test_conductor_eval_evaluator.py b/tests/test_conductor_eval_evaluator.py
index c8928b6..48c301b 100644
--- a/tests/test_conductor_eval_evaluator.py
+++ b/tests/test_conductor_eval_evaluator.py
@@ -2,7 +2,7 @@
import logging
import threading
import time
-from datetime import datetime
+from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
@@ -40,7 +40,9 @@ def test_ollama_discovery_returns_empty_when_unavailable(monkeypatch):
def fail_discovery():
raise RuntimeError("Ollama is unavailable")
- monkeypatch.setattr("conductor_eval.evaluator.ollama_api.get_model_list", fail_discovery)
+ monkeypatch.setattr(
+ "conductor_eval.evaluator.ollama_api.get_model_list", fail_discovery
+ )
assert Evaluator._discover_ollama_models() == []
@@ -83,7 +85,9 @@ def test_successful_evaluation_log_is_minimal(tmp_path):
assert "Saved result artifacts" not in log_contents
-def test_evaluations_keep_logs_and_artifacts_isolated_when_overlapping(monkeypatch, tmp_path):
+def test_evaluations_keep_logs_and_artifacts_isolated_when_overlapping(
+ monkeypatch, tmp_path
+):
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
resolution_barrier = threading.Barrier(2)
task_barrier = threading.Barrier(2)
@@ -159,7 +163,9 @@ def run_evaluation():
run_paths = list((tmp_path / "evaluations").iterdir())
assert len(run_paths) == 2
assert len({path.name for path in run_paths}) == 2
- assert all(path.name.startswith("20260726_123456_789012_same-name-") for path in run_paths)
+ assert all(
+ path.name.startswith("20260726_123456_789012_same-name-") for path in run_paths
+ )
for run_path in run_paths:
log_contents = (run_path / "run.log").read_text(encoding="utf-8")
@@ -168,7 +174,9 @@ def run_evaluation():
other_paths = [path for path in run_paths if path != run_path]
marker = config["models"][0][1]
other_marker = next(
- json.loads((path / "config.json").read_text(encoding="utf-8"))["models"][0][1]
+ json.loads((path / "config.json").read_text(encoding="utf-8"))["models"][0][
+ 1
+ ]
for path in other_paths
)
assert config["run_id"] == run_path.name
@@ -294,13 +302,20 @@ def test_save_results_uses_unique_safe_task_directory(tmp_path):
assert (result_dir / "loop.mid").exists()
legacy_filename = "output" + ".mid"
assert not (result_dir / legacy_filename).exists()
- assert json.loads((result_dir / "messages.json").read_text(encoding="utf-8")) == messages
+ assert (
+ json.loads((result_dir / "messages.json").read_text(encoding="utf-8"))
+ == messages
+ )
-def test_create_run_directory_returns_compact_authoritative_metadata(tmp_path, monkeypatch):
+def test_create_run_directory_returns_compact_authoritative_metadata(
+ tmp_path, monkeypatch
+):
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
- frozen_time = datetime(2026, 7, 28, 12, 34, 56, 789012)
- monkeypatch.setattr(evaluator_module, "datetime", SimpleNamespace(now=lambda: frozen_time))
+ frozen_time = datetime(2026, 7, 28, 12, 34, 56, 789012, tzinfo=timezone.utc)
+ monkeypatch.setattr(
+ evaluator_module, "datetime", SimpleNamespace(now=lambda _tz: frozen_time)
+ )
monkeypatch.setattr(
evaluator_module,
"uuid4",
@@ -320,7 +335,11 @@ def test_create_run_directory_fails_for_exact_collision(tmp_path, monkeypatch):
monkeypatch.setattr(
evaluator_module,
"datetime",
- SimpleNamespace(now=lambda: datetime(2026, 7, 28, 12, 34, 56, 789012)),
+ SimpleNamespace(
+ now=lambda _tz: datetime(
+ 2026, 7, 28, 12, 34, 56, 789012, tzinfo=timezone.utc
+ )
+ ),
)
monkeypatch.setattr(
evaluator_module,
@@ -526,7 +545,9 @@ def test_run_tests_marks_dangling_note_ineligible_under_default_checks(tmp_path)
@pytest.mark.parametrize("texture_test", ["monophony", "polyphony"])
-def test_run_tests_requires_completed_notes_for_texture_evidence(tmp_path, texture_test):
+def test_run_tests_requires_completed_notes_for_texture_evidence(
+ tmp_path, texture_test
+):
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
results = evaluator.run_tests(
@@ -583,7 +604,9 @@ def test_run_tests_always_includes_scale_when_callers_omit_it(tmp_path):
assert results["overall_status"] == "passed"
-def test_run_tests_classifies_checker_exceptions_as_ineligible_check_errors(monkeypatch, tmp_path):
+def test_run_tests_classifies_checker_exceptions_as_ineligible_check_errors(
+ monkeypatch, tmp_path
+):
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
def fail_scale_check(*args):
@@ -622,21 +645,28 @@ def fail_scale_check(*args):
({"overall_pass": False}, True),
],
)
-def test_overall_eligibility_contract_includes_only_valid_verdicts(test_results, expected):
+def test_overall_eligibility_contract_includes_only_valid_verdicts(
+ test_results, expected
+):
assert Evaluator._is_overall_eligible(test_results) is expected
@pytest.mark.parametrize(
("result", "expected"),
[
- ({"tests": {"overall_status": "rate_limited"}, "error": "throttled"}, "rate_limited"),
+ (
+ {"tests": {"overall_status": "rate_limited"}, "error": "throttled"},
+ "rate_limited",
+ ),
({"tests": {}, "error": "provider failed"}, "generation_error"),
({"tests": {"overall_pass": True}, "error": None}, "passed"),
({"tests": {"overall_pass": False}, "error": None}, "failed"),
({}, "failed"),
],
)
-def test_get_overall_status_preserves_status_and_supports_legacy_results(result, expected):
+def test_get_overall_status_preserves_status_and_supports_legacy_results(
+ result, expected
+):
assert get_overall_status(result) == expected
@@ -645,7 +675,9 @@ def test_evaluate_rejects_invalid_cloud_concurrency_before_output(tmp_path, valu
output_dir = tmp_path / "evaluations"
evaluator = Evaluator(output_dir=output_dir)
- with pytest.raises(ValueError, match="max_cloud_concurrency must be a positive integer"):
+ with pytest.raises(
+ ValueError, match="max_cloud_concurrency must be a positive integer"
+ ):
evaluator.evaluate(
prompts="melody",
roots=["C"],
@@ -819,7 +851,9 @@ def test_test_params_reject_unselected_test(tmp_path):
)
-def test_run_tests_rejects_unknown_checks_before_running_any_check(monkeypatch, tmp_path):
+def test_run_tests_rejects_unknown_checks_before_running_any_check(
+ monkeypatch, tmp_path
+):
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
def unexpected_scale_check(*args):
@@ -918,7 +952,9 @@ def test_summary_reports_latency_and_default_rates_for_ineligible_result(tmp_pat
assert summary["by_scale"]["major"]["pass_rate"] == 0.0
-def test_failed_generation_records_attempt_latency_and_contextual_log(monkeypatch, tmp_path):
+def test_failed_generation_records_attempt_latency_and_contextual_log(
+ monkeypatch, tmp_path
+):
class FailingAdapter:
def __init__(self, output_dir):
self.output_dir = output_dir
@@ -927,7 +963,9 @@ def generate(self, **kwargs):
raise RuntimeError("provider timed out")
monkeypatch.setattr("conductor_eval.evaluator.EvalEngineAdapter", FailingAdapter)
- monkeypatch.setattr("conductor_eval.evaluator.time.perf_counter", lambda: next(clock))
+ monkeypatch.setattr(
+ "conductor_eval.evaluator.time.perf_counter", lambda: next(clock)
+ )
clock = iter([100.0, 103.5])
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
task = {
@@ -960,7 +998,10 @@ def generate(self, **kwargs):
}
log_contents = (run_path / "run.log").read_text(encoding="utf-8")
assert "Task failed: task_id=task-prompt-0123456789abcdef-1" in log_contents
- assert "provider=OpenAI model=test-model root=C scale=major variation=standard" in log_contents
+ assert (
+ "provider=OpenAI model=test-model root=C scale=major variation=standard"
+ in log_contents
+ )
assert "error_type=RuntimeError" in log_contents
assert "Traceback:" in log_contents
assert "sensitive prompt" not in log_contents
@@ -975,7 +1016,9 @@ def generate(self, **kwargs):
return MidiFile(), [{"role": "assistant", "content": "loop"}], 0.125
monkeypatch.setattr("conductor_eval.evaluator.EvalEngineAdapter", SuccessfulAdapter)
- monkeypatch.setattr("conductor_eval.evaluator.time.perf_counter", lambda: next(clock))
+ monkeypatch.setattr(
+ "conductor_eval.evaluator.time.perf_counter", lambda: next(clock)
+ )
clock = iter([100.0, 101.25])
evaluator = Evaluator(output_dir=tmp_path / "evaluations")
monkeypatch.setattr(
diff --git a/uv.lock b/uv.lock
index d0a725e..2b5b010 100644
--- a/uv.lock
+++ b/uv.lock
@@ -339,7 +339,7 @@ requires-dist = [
{ name = "plotly", marker = "extra == 'dashboard'", specifier = "==6.0.1" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<9" },
{ name = "rich", specifier = ">=14.0.0" },
- { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.11,<1" },
+ { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.2" },
]
provides-extras = ["dashboard", "dev"]
@@ -455,7 +455,7 @@ name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+ { name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
wheels = [
@@ -1422,27 +1422,27 @@ wheels = [
[[package]]
name = "ruff"
-version = "0.16.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" },
- { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" },
- { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" },
- { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" },
- { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" },
- { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" },
- { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" },
- { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" },
- { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" },
- { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" },
- { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" },
- { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" },
- { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" },
- { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" },
- { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" },
- { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" },
- { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" },
+version = "0.16.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" },
+ { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" },
+ { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" },
+ { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" },
+ { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" },
+ { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" },
+ { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" },
]
[[package]]