From 6edfee45da1606bac440a3e07223700c3b0beb3f Mon Sep 17 00:00:00 2001 From: cxxxxxn Date: Sat, 18 Jul 2026 13:42:07 +0800 Subject: [PATCH 1/3] fix: remove dynamic exec from table loading Fixes #13 --- examples/agent/coml4vis.py | 15 +++++++++------ tests/test_coml4vis.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 tests/test_coml4vis.py diff --git a/examples/agent/coml4vis.py b/examples/agent/coml4vis.py index 008b9d5..7fecfd8 100644 --- a/examples/agent/coml4vis.py +++ b/examples/agent/coml4vis.py @@ -14,12 +14,15 @@ def read_table(name, url, format): - code = f"{name}_dataset = pd.read_csv('{url}')" - variable_description = {} - exec(code) - exec( - f"variable_description['{name}_dataset'] = describe_variable({name}_dataset, dataframe_format='{format}', pandas_description_config=dict(max_rows=10))" - ) + code = f"{name}_dataset = pd.read_csv({url!r})" + dataset = pd.read_csv(url) + variable_description = { + f"{name}_dataset": describe_variable( + dataset, + dataframe_format=format, + pandas_description_config=dict(max_rows=10), + ) + } return code, variable_description diff --git a/tests/test_coml4vis.py b/tests/test_coml4vis.py new file mode 100644 index 0000000..13e5719 --- /dev/null +++ b/tests/test_coml4vis.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from unittest.mock import Mock + +from examples.agent.coml4vis import read_table + + +def test_read_table_without_dynamic_execution(monkeypatch): + dataset = object() + read_csv = Mock(return_value=dataset) + describe_variable = Mock(return_value={"summary": "ok"}) + monkeypatch.setattr("examples.agent.coml4vis.pd.read_csv", read_csv) + monkeypatch.setattr( + "examples.agent.coml4vis.describe_variable", describe_variable + ) + + code, variable_description = read_table("sales", "/tmp/data.csv", "coml") + + assert code == "sales_dataset = pd.read_csv('/tmp/data.csv')" + assert variable_description == {"sales_dataset": {"summary": "ok"}} + read_csv.assert_called_once_with("/tmp/data.csv") + describe_variable.assert_called_once_with( + dataset, + dataframe_format="coml", + pandas_description_config={"max_rows": 10}, + ) \ No newline at end of file From f26928a604eb0b19faff89f5c8707683bac4a7c4 Mon Sep 17 00:00:00 2001 From: cxxxxxn Date: Sat, 18 Jul 2026 13:42:22 +0800 Subject: [PATCH 2/3] fix: declare optional agent dependencies Fixes #18 --- README.md | 11 +++++++++++ examples/agent/__init__.py | 19 ++++++++++++++++--- examples/evaluate.py | 7 ++++++- pyproject.toml | 12 +++++++++++- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 46f8410..d5689b6 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,17 @@ pip install --upgrade vis-evaluator # or `git clone https://github.com/microsoft/VisEval.git && cd VisEval && pip install --upgrade -e .` ``` +To run the generation agents in `examples/agent`, install their optional +dependencies as well: + +```bash +pip install --upgrade "vis-evaluator[agents]" +# or, from a clone: `pip install --upgrade -e ".[agents]"` +``` + +The `mlcopilot` distribution provides the `coml` Python package used by +`CoML4VIS`. + ### Download Benchmark Dataset To access the dataset, please follow these steps: diff --git a/examples/agent/__init__.py b/examples/agent/__init__.py index 7b12903..9fc615c 100644 --- a/examples/agent/__init__.py +++ b/examples/agent/__init__.py @@ -1,6 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from .chat2vis import Chat2vis -from .coml4vis import CoML4VIS -from .lida import Lida +from importlib import import_module + +__all__ = ["Chat2vis", "CoML4VIS", "Lida"] + +_AGENT_MODULES = { + "Chat2vis": ".chat2vis", + "CoML4VIS": ".coml4vis", + "Lida": ".lida", +} + + +def __getattr__(name: str): + if name not in _AGENT_MODULES: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = import_module(_AGENT_MODULES[name], __name__) + return getattr(module, name) diff --git a/examples/evaluate.py b/examples/evaluate.py index 135f59d..944c54d 100644 --- a/examples/evaluate.py +++ b/examples/evaluate.py @@ -5,7 +5,6 @@ from pathlib import Path import dotenv -from agent import Chat2vis, CoML4VIS, Lida from viseval import Dataset, Evaluator @@ -55,10 +54,16 @@ def configure_llm(model: str, agent: str): def config_agent(agent: str, model: str, config: dict): llm = configure_llm(model, agent) if agent == "coml4vis": + from agent import CoML4VIS + return CoML4VIS(llm, config) elif agent == "chat2vis": + from agent import Chat2vis + return Chat2vis(llm) elif agent == "lida": + from agent import Lida + return Lida(llm) else: raise ValueError(f"Unknown agent {agent}") diff --git a/pyproject.toml b/pyproject.toml index 7cffb5c..7aa56f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "vis-evaluator" version = "0.0.4" requires-python = ">=3.10" dependencies = [ - "langchain", + "langchain<1", "python-dotenv", "numpy", "pandas", @@ -25,6 +25,13 @@ Issues = "https://github.com/microsoft/VisEval/issues" Source = "https://github.com/microsoft/VisEval" [project.optional-dependencies] +agents = [ + "mlcopilot", + "lida", + "langchain-community<0.4", + "langchain-google-genai<3", + "langchain-openai<1", +] dev = [ "pytest", "flake8", @@ -34,6 +41,9 @@ dev = [ "hatch", "mlcopilot", "lida", + "langchain-community<0.4", + "langchain-google-genai<3", + "langchain-openai<1", ] From c017516a3b1db9525d7def66b17c93da8ace6f05 Mon Sep 17 00:00:00 2001 From: cxxxxxn Date: Sat, 18 Jul 2026 13:42:36 +0800 Subject: [PATCH 3/3] fix: handle incomplete chart encodings Fixes #19 --- tests/test_order_check.py | 35 +++++++++++++++++++++++++++++++++++ viseval/check/order_check.py | 16 ++++++++++++++-- viseval/evaluate.py | 1 - 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/tests/test_order_check.py b/tests/test_order_check.py index 31fc995..e303d22 100644 --- a/tests/test_order_check.py +++ b/tests/test_order_check.py @@ -185,3 +185,38 @@ def test_order_check_bar_465(): chart_info, ground_truth, query_meta[0]["sort_by"] ) assert answer is False + + +@pytest.mark.parametrize( + ("chart_info", "expected_rationale"), + [ + ( + { + "encoding": {"y": {"scale": {}}}, + "channel_map": {"x": "x", "y": "y"}, + }, + "Missing x encoding.", + ), + ( + { + "encoding": {"x": {}, "y": {"scale": {}}}, + "channel_map": {"x": "x", "y": "y"}, + }, + "Missing scale for x encoding.", + ), + ( + { + "encoding": {"x": {"scale": {}}, "y": {"scale": {}}}, + "channel_map": {"y": "y"}, + }, + "Missing x channel mapping.", + ), + ], +) +def test_order_check_rejects_incomplete_chart_info(chart_info, expected_rationale): + ground_truth = {"sort": {"channel": "x", "order": "descending"}} + + answer, rationale = order_check(chart_info, ground_truth, "axis") + + assert answer is False + assert rationale == expected_rationale diff --git a/viseval/check/order_check.py b/viseval/check/order_check.py index a55a101..0e382f2 100644 --- a/viseval/check/order_check.py +++ b/viseval/check/order_check.py @@ -5,7 +5,6 @@ def order_check(chart_info: dict, ground_truth: dict, sort_by: str): order = ground_truth["sort"] encoding = chart_info["encoding"] - data = chart_info["data"] channel_map = chart_info["channel_map"] if order is not None: @@ -13,12 +12,25 @@ def order_check(chart_info: dict, ground_truth: dict, sort_by: str): if sort_by == "axis": order_channel = order["channel"] else: - order_channel = channel_map[order["channel"]] + source_channel = order["channel"] + if source_channel not in channel_map: + return False, f"Missing {source_channel} channel mapping." + order_channel = channel_map[source_channel] other_channel = "y" if order_channel == "x" else "x" + for channel in (order_channel, other_channel): + if channel not in encoding: + return False, f"Missing {channel} encoding." + if "scale" not in encoding[channel]: + return False, f"Missing scale for {channel} encoding." + + if order_channel not in channel_map: + return False, f"Missing {order_channel} channel mapping." + order_channel_scale = encoding[order_channel]["scale"] other_channel_scale = encoding[other_channel]["scale"] + data = chart_info["data"] # origin channel if ( diff --git a/viseval/evaluate.py b/viseval/evaluate.py index 524df70..52fa8f4 100644 --- a/viseval/evaluate.py +++ b/viseval/evaluate.py @@ -395,7 +395,6 @@ def legality_check(self, context, ground_truth) -> list[CheckResult]: results.append(chart_type_check_result) results.append(data_check_result) if data_check_result.answer and ground_truth["vis_obj"]["sort"] is not None: - self.order_check(context, ground_truth) results.append(self.order_check(context, ground_truth)) return results