fix: handle incomplete chart encodings, optional agent deps, and unsafe table loading

README.md

@@ -20,6 +20,17 @@ pip install --upgrade vis-evaluator
 # or `git clone https://github.com/microsoft/VisEval.git && cd VisEval && pip install --upgrade -e .`
 ```
 
+To run the generation agents in `examples/agent`, install their optional
+dependencies as well:
+
+```bash
+pip install --upgrade "vis-evaluator[agents]"
+# or, from a clone: `pip install --upgrade -e ".[agents]"`
+```
+
+The `mlcopilot` distribution provides the `coml` Python package used by
+`CoML4VIS`.
+
 ### Download Benchmark Dataset
 To access the dataset, please follow these steps:
 

examples/agent/__init__.py

@@ -1,6 +1,19 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-from .chat2vis import Chat2vis
-from .coml4vis import CoML4VIS
-from .lida import Lida
+from importlib import import_module
+
+__all__ = ["Chat2vis", "CoML4VIS", "Lida"]
+
+_AGENT_MODULES = {
+    "Chat2vis": ".chat2vis",
+    "CoML4VIS": ".coml4vis",
+    "Lida": ".lida",
+}
+
+
+def __getattr__(name: str):
+    if name not in _AGENT_MODULES:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module = import_module(_AGENT_MODULES[name], __name__)
+    return getattr(module, name)

examples/agent/coml4vis.py

@@ -14,12 +14,15 @@
 
 
 def read_table(name, url, format):
-    code = f"{name}_dataset = pd.read_csv('{url}')"
-    variable_description = {}
-    exec(code)
-    exec(
-        f"variable_description['{name}_dataset'] = describe_variable({name}_dataset, dataframe_format='{format}', pandas_description_config=dict(max_rows=10))"
-    )
+    code = f"{name}_dataset = pd.read_csv({url!r})"
+    dataset = pd.read_csv(url)
+    variable_description = {
+        f"{name}_dataset": describe_variable(
+            dataset,
+            dataframe_format=format,
+            pandas_description_config=dict(max_rows=10),
+        )
+    }
     return code, variable_description
 
 

examples/evaluate.py

@@ -5,7 +5,6 @@
 from pathlib import Path
 
 import dotenv
-from agent import Chat2vis, CoML4VIS, Lida
 
 from viseval import Dataset, Evaluator
 
@@ -55,10 +54,16 @@ def configure_llm(model: str, agent: str):
 def config_agent(agent: str, model: str, config: dict):
     llm = configure_llm(model, agent)
     if agent == "coml4vis":
+        from agent import CoML4VIS
+
         return CoML4VIS(llm, config)
     elif agent == "chat2vis":
+        from agent import Chat2vis
+
         return Chat2vis(llm)
     elif agent == "lida":
+        from agent import Lida
+
         return Lida(llm)
     else:
         raise ValueError(f"Unknown agent {agent}")

pyproject.toml

@@ -3,7 +3,7 @@ name = "vis-evaluator"
 version = "0.0.4"
 requires-python = ">=3.10"
 dependencies = [
-    "langchain",
+    "langchain<1",
     "python-dotenv",
     "numpy",
     "pandas",
@@ -25,6 +25,13 @@ Issues = "https://github.com/microsoft/VisEval/issues"
 Source = "https://github.com/microsoft/VisEval"
 
 [project.optional-dependencies]
+agents = [
+    "mlcopilot",
+    "lida",
+    "langchain-community<0.4",
+    "langchain-google-genai<3",
+    "langchain-openai<1",
+]
 dev = [
     "pytest",
     "flake8",
@@ -34,6 +41,9 @@ dev = [
     "hatch",
     "mlcopilot",
     "lida",
+    "langchain-community<0.4",
+    "langchain-google-genai<3",
+    "langchain-openai<1",
 ]
 
 

tests/test_coml4vis.py

@@ -0,0 +1,27 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+from unittest.mock import Mock
+
+from examples.agent.coml4vis import read_table
+
+
+def test_read_table_without_dynamic_execution(monkeypatch):
+    dataset = object()
+    read_csv = Mock(return_value=dataset)
+    describe_variable = Mock(return_value={"summary": "ok"})
+    monkeypatch.setattr("examples.agent.coml4vis.pd.read_csv", read_csv)
+    monkeypatch.setattr(
+        "examples.agent.coml4vis.describe_variable", describe_variable
+    )
+
+    code, variable_description = read_table("sales", "/tmp/data.csv", "coml")
+
+    assert code == "sales_dataset = pd.read_csv('/tmp/data.csv')"
+    assert variable_description == {"sales_dataset": {"summary": "ok"}}
+    read_csv.assert_called_once_with("/tmp/data.csv")
+    describe_variable.assert_called_once_with(
+        dataset,
+        dataframe_format="coml",
+        pandas_description_config={"max_rows": 10},
+    )

tests/test_order_check.py

@@ -185,3 +185,38 @@ def test_order_check_bar_465():
             chart_info, ground_truth, query_meta[0]["sort_by"]
         )
         assert answer is False
+
+
+@pytest.mark.parametrize(
+    ("chart_info", "expected_rationale"),
+    [
+        (
+            {
+                "encoding": {"y": {"scale": {}}},
+                "channel_map": {"x": "x", "y": "y"},
+            },
+            "Missing x encoding.",
+        ),
+        (
+            {
+                "encoding": {"x": {}, "y": {"scale": {}}},
+                "channel_map": {"x": "x", "y": "y"},
+            },
+            "Missing scale for x encoding.",
+        ),
+        (
+            {
+                "encoding": {"x": {"scale": {}}, "y": {"scale": {}}},
+                "channel_map": {"y": "y"},
+            },
+            "Missing x channel mapping.",
+        ),
+    ],
+)
+def test_order_check_rejects_incomplete_chart_info(chart_info, expected_rationale):
+    ground_truth = {"sort": {"channel": "x", "order": "descending"}}
+
+    answer, rationale = order_check(chart_info, ground_truth, "axis")
+
+    assert answer is False
+    assert rationale == expected_rationale

viseval/check/order_check.py

@@ -5,20 +5,32 @@
 def order_check(chart_info: dict, ground_truth: dict, sort_by: str):
     order = ground_truth["sort"]
     encoding = chart_info["encoding"]
-    data = chart_info["data"]
     channel_map = chart_info["channel_map"]
 
     if order is not None:
         # bar, line
         if sort_by == "axis":
             order_channel = order["channel"]
         else:
-            order_channel = channel_map[order["channel"]]
+            source_channel = order["channel"]
+            if source_channel not in channel_map:
+                return False, f"Missing {source_channel} channel mapping."
+            order_channel = channel_map[source_channel]
 
         other_channel = "y" if order_channel == "x" else "x"
 
+        for channel in (order_channel, other_channel):
+            if channel not in encoding:
+                return False, f"Missing {channel} encoding."
+            if "scale" not in encoding[channel]:
+                return False, f"Missing scale for {channel} encoding."
+
+        if order_channel not in channel_map:
+            return False, f"Missing {order_channel} channel mapping."
+
         order_channel_scale = encoding[order_channel]["scale"]
         other_channel_scale = encoding[other_channel]["scale"]
+        data = chart_info["data"]
 
         # origin channel
         if (

viseval/evaluate.py

@@ -395,7 +395,6 @@ def legality_check(self, context, ground_truth) -> list[CheckResult]:
             results.append(chart_type_check_result)
             results.append(data_check_result)
             if data_check_result.answer and ground_truth["vis_obj"]["sort"] is not None:
-                self.order_check(context, ground_truth)
                 results.append(self.order_check(context, ground_truth))
 
         return results