diff --git a/.gitignore b/.gitignore index b8bcbda1..fa4f8942 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ checkpoints/ external/ *.ipynb +!examples/notebooks/trace_quickstart.ipynb *.parquet *.sqlite *.sqlite3 diff --git a/README.md b/README.md index d52b44a5..c8851fd7 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ contains 1,000 tasks across 277 scenes and 11 visual domains.

Project Page + Live Demo arXiv Hugging Face Collection Dataset @@ -18,6 +19,42 @@ contains 1,000 tasks across 277 scenes and 11 visual domains. ![Trace examples across charts, games, geometry, graphs, icons, illustrations, pages, physics, puzzles, symbolic reasoning, and 3D scenes](docs/assets/paper-domain-montage/trace-paper-domain-montage.png) +## Try Trace + +

+ Open Trace Space + Open in Colab +

+ +Trace is for researchers and engineers building verifiable VLM post-training, +synthetic-data, and evaluation pipelines. + +Explore all 1,000 tasks in the browser, including their deterministic images, +prompts, typed supervision, reward contracts, annotation overlays, and public +execution traces. Or generate the default example locally from the exact +revision used by the demo: + +```bash +python -m pip install \ + "trace-tasks @ git+https://github.com/maveryn/trace.git@bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7" +``` + +```python +from trace_tasks import generate_task + +sample = generate_task( + "task_geometry__graph_paper__polygon_area_value", + seed=42, + max_attempts=100, +) +sample.image.save("trace-example.png") +print(sample.prompt) +print(sample.answer_gt.to_dict()) +print(sample.annotation_gt.to_dict()) +``` + +![The Trace Space showing a generated task, its public annotation overlay, prompt, and deterministic selectors](docs/assets/examples/trace-space-quickstart.png) + ## How Trace Works Trace organizes visual reasoning as `domain → scene grammar → task program`. diff --git a/docs/README.md b/docs/README.md index 8fb3ae3f..209f0552 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,7 +27,8 @@ hide:

- Code + Live Demo + Code Paper Research Collection @@ -68,6 +69,51 @@ hide: +## Try Trace + +
+ Open Trace Space + Open in Colab +
+ +Trace is for researchers and engineers building verifiable VLM post-training, +synthetic-data, and evaluation pipelines. + +Explore all 1,000 tasks in the browser, including their deterministic images, +prompts, typed supervision, reward contracts, annotation overlays, and public +execution traces. Or generate the default example locally from the exact +revision used by the demo: + +```bash +python -m pip install \ + "trace-tasks @ git+https://github.com/maveryn/trace.git@bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7" +``` + +```python +from trace_tasks import generate_task + +sample = generate_task( + "task_geometry__graph_paper__polygon_area_value", + seed=42, + max_attempts=100, +) +sample.image.save("trace-example.png") +print(sample.prompt) +print(sample.answer_gt.to_dict()) +print(sample.annotation_gt.to_dict()) +``` + +
+ The Trace Space showing a generated task, its public annotation overlay, prompt, and deterministic selectors. +
+ The public Space exposes the same deterministic task and verifier contracts + as the Python package. +
+
+ ## How Trace works Trace organizes visual reasoning as `domain → scene grammar → task program`. diff --git a/docs/assets/examples/trace-space-quickstart.png b/docs/assets/examples/trace-space-quickstart.png new file mode 100644 index 00000000..0a78be59 Binary files /dev/null and b/docs/assets/examples/trace-space-quickstart.png differ diff --git a/examples/huggingface_space/README.md b/examples/huggingface_space/README.md new file mode 100644 index 00000000..be43d91c --- /dev/null +++ b/examples/huggingface_space/README.md @@ -0,0 +1,58 @@ +--- +title: Trace Task Explorer +emoji: 🔎 +colorFrom: blue +colorTo: indigo +sdk: gradio +sdk_version: 6.20.0 +python_version: "3.12" +app_file: app.py +pinned: false +license: apache-2.0 +short_description: Generate and inspect 1,000 grounded visual-reasoning tasks. +suggested_hardware: cpu-basic +models: + - maveryn/trace-qwen2.5-vl-3b + - maveryn/trace-qwen2.5-vl-7b +datasets: + - maveryn/trace +tags: + - visual-reasoning + - synthetic-data + - reinforcement-learning + - rlvr + - multimodal +--- + +# Trace Task Explorer + +This Space generates any of Trace's 1,000 deterministic, grounded +visual-reasoning tasks. Select a domain, scene, task, and seed to inspect the +rendered image, prompt, typed supervision, public annotation overlay, exact +reward contract, and sanitized public execution trace. + +The Space is CPU-only and may need a short cold start. It accepts no uploads and +stores no generated examples. The package dependency is pinned to the exact +source revision shown in the **Reproduce** tab. + +- [Source repository](https://github.com/maveryn/trace) +- [Documentation](https://maveryn.github.io/trace/) +- [Paper](https://arxiv.org/abs/2607.19790) +- [Dataset](https://huggingface.co/datasets/maveryn/trace) +- [Hugging Face collection](https://huggingface.co/collections/maveryn/trace-6a604291b4be4ed6399b9f24) +- [3B checkpoint](https://huggingface.co/maveryn/trace-qwen2.5-vl-3b) +- [7B checkpoint](https://huggingface.co/maveryn/trace-qwen2.5-vl-7b) + +## Citation + +```bibtex +@misc{alam2026trace, + title = {Trace: A Taxonomy-Guided Environment for Multidomain Visual Reasoning}, + author = {Alam, Md Tanvirul}, + year = {2026}, + eprint = {2607.19790}, + archivePrefix = {arXiv}, + primaryClass = {cs.CV}, + url = {https://arxiv.org/abs/2607.19790} +} +``` diff --git a/examples/huggingface_space/app.py b/examples/huggingface_space/app.py new file mode 100644 index 00000000..9675f776 --- /dev/null +++ b/examples/huggingface_space/app.py @@ -0,0 +1,311 @@ +"""Gradio entry point for the public Trace task explorer.""" + +from __future__ import annotations + +import os + +os.environ.setdefault("GRADIO_ANALYTICS_ENABLED", "False") + +import gradio as gr + +from trace_demo import ( + DEFAULT_DOMAIN, + DEFAULT_SCENE_ID, + DEFAULT_SEED, + DEFAULT_TASK_ID, + MAX_SEED, + build_catalog, + generate_demo, + load_presets, + sample_random_selection, +) + +CATALOG = build_catalog() +PRESETS = load_presets() + +_CSS = """ +.trace-shell {max-width: 1440px; margin: 0 auto;} +.trace-kicker {letter-spacing: .12em; text-transform: uppercase; font-size: .78rem; + color: var(--body-text-color-subdued);} +.trace-title h1 {margin-bottom: .25rem;} +.trace-title p {max-width: 900px; font-size: 1.02rem;} +.trace-stat {border: 1px solid var(--border-color-primary); border-radius: 12px; + padding: .8rem 1rem; background: var(--background-fill-secondary);} +.trace-stat strong {font-size: 1.35rem; display: block;} +.trace-run {min-height: 48px;} +.trace-note {font-size: .9rem; color: var(--body-text-color-subdued);} +""" + + +def _scene_update(domain: str): + scenes = CATALOG.scenes(domain) + scene_id = scenes[0] + return ( + gr.Dropdown(choices=list(scenes), value=scene_id), + gr.Dropdown( + choices=list(CATALOG.tasks(domain, scene_id)), + value=CATALOG.tasks(domain, scene_id)[0], + ), + ) + + +def _task_update(domain: str, scene_id: str): + tasks = CATALOG.tasks(domain, scene_id) + return gr.Dropdown(choices=list(tasks), value=tasks[0]) + + +def _preset_update(preset_index: str): + try: + preset = PRESETS[int(preset_index)] + except (IndexError, TypeError, ValueError) as exc: + raise gr.Error("Choose one of the curated Trace presets.") from exc + return ( + gr.Dropdown(choices=list(CATALOG.domains), value=preset.domain), + gr.Dropdown( + choices=list(CATALOG.scenes(preset.domain)), + value=preset.scene_id, + ), + gr.Dropdown( + choices=list(CATALOG.tasks(preset.domain, preset.scene_id)), + value=preset.task_id, + ), + preset.seed, + ) + + +def _run_generation(task_id: str, seed: int): + try: + result = generate_demo(task_id, seed, catalog=CATALOG) + except (KeyError, TypeError, ValueError, RuntimeError) as exc: + raise gr.Error(f"Trace could not generate that selection: {str(exc)[:300]}") from exc + return ( + result.original_image, + result.annotation_overlay, + result.prompt, + result.ground_truth, + result.reward_contract, + result.trace_summary, + result.public_trace, + result.reproduction, + result.links_markdown, + ) + + +def _random_question(): + selection = sample_random_selection(CATALOG) + return ( + gr.Dropdown( + choices=list(CATALOG.domains), + value=selection.domain, + ), + gr.Dropdown( + choices=list(CATALOG.scenes(selection.domain)), + value=selection.scene_id, + ), + gr.Dropdown( + choices=list(CATALOG.tasks(selection.domain, selection.scene_id)), + value=selection.task_id, + ), + selection.seed, + *_run_generation(selection.task_id, selection.seed), + ) + + +with gr.Blocks( + title="Trace · Grounded visual reasoning", + analytics_enabled=False, + fill_width=True, +) as demo: + with gr.Column(elem_classes="trace-shell"): + gr.HTML('
Grounded visual reasoning · deterministic by design
') + gr.Markdown( + """ +# Explore Trace + +Generate any of Trace's **1,000 tasks** across **277 scenes** and **11 visual +domains**. Every image, prompt, typed answer, annotation, reward contract, and +public execution trace comes from the same deterministic state. +""", + elem_classes="trace-title", + ) + + with gr.Row(equal_height=True): + gr.HTML("
1,000tasks
") + gr.HTML("
277scenes
") + gr.HTML("
11domains
") + + with gr.Row(): + with gr.Column(scale=7): + with gr.Row(): + domain = gr.Dropdown( + choices=list(CATALOG.domains), + value=DEFAULT_DOMAIN, + label="1 · Domain", + interactive=True, + ) + scene_id = gr.Dropdown( + choices=list(CATALOG.scenes(DEFAULT_DOMAIN)), + value=DEFAULT_SCENE_ID, + label="2 · Scene", + interactive=True, + ) + task_id = gr.Dropdown( + choices=list(CATALOG.tasks(DEFAULT_DOMAIN, DEFAULT_SCENE_ID)), + value=DEFAULT_TASK_ID, + label="3 · Task (searchable)", + filterable=True, + interactive=True, + ) + with gr.Column(scale=3): + seed = gr.Number( + value=DEFAULT_SEED, + minimum=0, + maximum=MAX_SEED, + precision=0, + label="Seed", + interactive=True, + ) + with gr.Row(): + randomize = gr.Button("Random question", variant="secondary") + generate = gr.Button( + "Generate task", + variant="primary", + elem_classes="trace-run", + ) + gr.Markdown( + "Inputs are limited to a registered task and integer seed. " + "Generation uses `max_attempts=100`.", + elem_classes="trace-note", + ) + + preset = gr.Dropdown( + choices=[ + (item.label, str(index)) + for index, item in enumerate(PRESETS) + ], + value=None, + label="Curated gallery · 22 deterministic presets, two per domain", + filterable=True, + interactive=True, + ) + + with gr.Tabs(): + with gr.Tab("Problem"): + with gr.Row(): + original_image = gr.Image( + label="Generated image", + type="pil", + format="png", + interactive=False, + ) + annotation_overlay = gr.Image( + label="Public annotation overlay", + type="pil", + format="png", + interactive=False, + ) + prompt = gr.Textbox( + label="Prompt", + lines=4, + interactive=False, + buttons=["copy"], + ) + with gr.Tab("Ground truth"): + with gr.Row(): + ground_truth = gr.JSON(label="Typed answer and annotation") + reward_contract = gr.JSON(label="Reward contract") + with gr.Tab("Execution trace"): + with gr.Row(): + trace_summary = gr.JSON(label="Trace summary") + public_trace = gr.JSON( + label="Full public trace", + open=False, + ) + with gr.Tab("Reproduce"): + reproduction = gr.Code( + label="Exact-revision reproduction", + language="shell", + interactive=False, + lines=13, + ) + + links = gr.Markdown( + "Choose a task and seed, then select **Generate task**.", + ) + gr.Markdown( + """ +Trace uses metadata contracts—not pixels—as verifier ground truth. The overlay +is an inspection aid; the typed payload and reward contract are authoritative. + +[GitHub](https://github.com/maveryn/trace) · +[Documentation](https://maveryn.github.io/trace/) · +[Dataset](https://huggingface.co/datasets/maveryn/trace) · +[Paper](https://arxiv.org/abs/2607.19790) +""", + elem_classes="trace-note", + ) + + domain.input( + _scene_update, + inputs=domain, + outputs=[scene_id, task_id], + api_name=False, + concurrency_limit=1, + ) + scene_id.input( + _task_update, + inputs=[domain, scene_id], + outputs=task_id, + api_name=False, + concurrency_limit=1, + ) + randomize.click( + _random_question, + outputs=[ + domain, + scene_id, + task_id, + seed, + original_image, + annotation_overlay, + prompt, + ground_truth, + reward_contract, + trace_summary, + public_trace, + reproduction, + links, + ], + api_name=False, + concurrency_limit=1, + ) + preset.change( + _preset_update, + inputs=preset, + outputs=[domain, scene_id, task_id, seed], + api_name=False, + concurrency_limit=1, + ) + generate.click( + _run_generation, + inputs=[task_id, seed], + outputs=[ + original_image, + annotation_overlay, + prompt, + ground_truth, + reward_contract, + trace_summary, + public_trace, + reproduction, + links, + ], + api_name=False, + concurrency_limit=1, + ) + +demo.queue(default_concurrency_limit=1, max_size=32) + + +if __name__ == "__main__": + demo.launch(css=_CSS, footer_links=[]) diff --git a/examples/huggingface_space/overlay.py b/examples/huggingface_space/overlay.py new file mode 100644 index 00000000..3cbc5061 --- /dev/null +++ b/examples/huggingface_space/overlay.py @@ -0,0 +1,187 @@ +"""Render review-only overlays from Trace's public image annotations.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from PIL import Image, ImageDraw, ImageFont + +PUBLIC_ANNOTATION_TYPES = frozenset( + { + "bbox", + "bbox_map", + "bbox_sequence", + "bbox_set", + "bbox_set_map", + "point", + "point_map", + "point_sequence", + "point_set", + "point_set_map", + "segment", + "segment_set", + } +) + +_COLORS = ( + (229, 57, 53, 235), + (30, 136, 229, 235), + (0, 137, 123, 235), + (251, 140, 0, 235), + (142, 36, 170, 235), +) + + +def render_annotation_overlay( + source_image: Image.Image, + annotation_gt: Mapping[str, Any], +) -> Image.Image: + """Return an RGB image with a public annotation drawn over the source.""" + + image = source_image.convert("RGBA").copy() + draw = ImageDraw.Draw(image, "RGBA") + annotation_type = str(annotation_gt.get("type", "")).strip() + if annotation_type not in PUBLIC_ANNOTATION_TYPES: + raise ValueError(f"unsupported public annotation type: {annotation_type!r}") + + items = list(_annotation_items(annotation_type, annotation_gt.get("value"))) + for index, (label, geometry_kind, geometry) in enumerate(items): + color = _COLORS[index % len(_COLORS)] + if geometry_kind == "bbox": + _draw_bbox(draw, geometry, color=color, label=label) + elif geometry_kind == "point": + _draw_point(draw, geometry, color=color, label=label) + elif geometry_kind == "segment": + _draw_segment(draw, geometry, color=color, label=label) + + if not items: + _draw_empty_annotation_badge(draw, annotation_type) + return image.convert("RGB") + + +def _annotation_items(annotation_type: str, value: Any): + if annotation_type == "bbox": + yield "answer", "bbox", value + elif annotation_type in {"bbox_set", "bbox_sequence"} and _sequence(value): + for index, bbox in enumerate(value): + yield str(index + 1), "bbox", bbox + elif annotation_type == "bbox_map" and isinstance(value, Mapping): + for key, bbox in sorted(value.items(), key=lambda item: str(item[0])): + yield str(key), "bbox", bbox + elif annotation_type == "bbox_set_map" and isinstance(value, Mapping): + for key, boxes in sorted(value.items(), key=lambda item: str(item[0])): + if not _sequence(boxes): + continue + for index, bbox in enumerate(boxes): + yield f"{key}:{index + 1}", "bbox", bbox + elif annotation_type == "point": + yield "answer", "point", value + elif annotation_type in {"point_set", "point_sequence"} and _sequence(value): + for index, point in enumerate(value): + yield str(index + 1), "point", point + elif annotation_type == "point_map" and isinstance(value, Mapping): + for key, point in sorted(value.items(), key=lambda item: str(item[0])): + yield str(key), "point", point + elif annotation_type == "point_set_map" and isinstance(value, Mapping): + for key, points in sorted(value.items(), key=lambda item: str(item[0])): + if not _sequence(points): + continue + for index, point in enumerate(points): + yield f"{key}:{index + 1}", "point", point + elif annotation_type == "segment": + yield "answer", "segment", value + elif annotation_type == "segment_set" and _sequence(value): + for index, segment in enumerate(value): + yield str(index + 1), "segment", segment + + +def _draw_bbox( + draw: ImageDraw.ImageDraw, + value: Any, + *, + color: tuple[int, ...], + label: str, +) -> None: + if not _numeric_sequence(value, 4): + return + x0, y0, x1, y1 = (float(item) for item in value) + draw.rectangle((x0, y0, x1, y1), outline=color, width=4) + _draw_label(draw, (x0, y0), label, color) + + +def _draw_point( + draw: ImageDraw.ImageDraw, + value: Any, + *, + color: tuple[int, ...], + label: str, +) -> None: + if not _numeric_sequence(value, 2): + return + x, y = (float(item) for item in value) + radius = 7.0 + draw.ellipse((x - radius, y - radius, x + radius, y + radius), fill=color) + _draw_label(draw, (x + radius, y - radius), label, color) + + +def _draw_segment( + draw: ImageDraw.ImageDraw, + value: Any, + *, + color: tuple[int, ...], + label: str, +) -> None: + if not _sequence(value) or len(value) != 2: + return + if not _numeric_sequence(value[0], 2) or not _numeric_sequence(value[1], 2): + return + points = [(float(point[0]), float(point[1])) for point in value] + draw.line(points, fill=color, width=5) + _draw_label(draw, points[0], label, color) + + +def _draw_label( + draw: ImageDraw.ImageDraw, + point: tuple[float, float], + label: str, + color: tuple[int, ...], +) -> None: + text = str(label)[:48] + if not text: + return + font = ImageFont.load_default() + left, top = max(0.0, float(point[0])), max(0.0, float(point[1]) - 16.0) + right = left + max(18.0, float(draw.textlength(text, font=font)) + 8.0) + draw.rectangle((left, top, right, top + 16.0), fill=color) + draw.text((left + 4.0, top + 2.0), text, fill=(255, 255, 255, 255), font=font) + + +def _draw_empty_annotation_badge( + draw: ImageDraw.ImageDraw, + annotation_type: str, +) -> None: + _draw_label( + draw, + (8.0, 22.0), + f"{annotation_type}: empty witness", + (69, 90, 100, 235), + ) + + +def _sequence(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance( + value, (str, bytes, bytearray) + ) + + +def _numeric_sequence(value: Any, length: int) -> bool: + if not _sequence(value) or len(value) != length: + return False + return all( + isinstance(item, (int, float)) and not isinstance(item, bool) + for item in value + ) + + +__all__ = ["PUBLIC_ANNOTATION_TYPES", "render_annotation_overlay"] diff --git a/examples/huggingface_space/packages.txt b/examples/huggingface_space/packages.txt new file mode 100644 index 00000000..23225089 --- /dev/null +++ b/examples/huggingface_space/packages.txt @@ -0,0 +1 @@ +libcairo2 diff --git a/examples/huggingface_space/presets.json b/examples/huggingface_space/presets.json new file mode 100644 index 00000000..bca476b7 --- /dev/null +++ b/examples/huggingface_space/presets.json @@ -0,0 +1,137 @@ +{ + "schema_version": "trace_space_presets_v1", + "presets": [ + { + "domain": "charts", + "scene_id": "dashboard", + "task_id": "task_charts__dashboard__category_total_extremum_label", + "seed": 3201 + }, + { + "domain": "charts", + "scene_id": "radial_sankey", + "task_id": "task_charts__radial_sankey__dominant_endpoint_label", + "seed": 3202 + }, + { + "domain": "games", + "scene_id": "chess", + "task_id": "task_games__chess__colored_piece_kind_count", + "seed": 3301 + }, + { + "domain": "games", + "scene_id": "pacman", + "task_id": "task_games__pacman__route_score_value", + "seed": 3302 + }, + { + "domain": "geometry", + "scene_id": "graph_paper", + "task_id": "task_geometry__graph_paper__polygon_area_value", + "seed": 3401 + }, + { + "domain": "geometry", + "scene_id": "solid_revolution", + "task_id": "task_geometry__solid_revolution__revolution_double_cone_volume_value", + "seed": 3402 + }, + { + "domain": "graph", + "scene_id": "node_link", + "task_id": "task_graph__node_link__shortest_path_length", + "seed": 3501 + }, + { + "domain": "graph", + "scene_id": "metro", + "task_id": "task_graph__metro__shortest_path_length", + "seed": 3502 + }, + { + "domain": "icons", + "scene_id": "mirror_grid", + "task_id": "task_icons__mirror_grid__missing_mirror_cell_label", + "seed": 3601 + }, + { + "domain": "icons", + "scene_id": "wallpaper_panels", + "task_id": "task_icons__wallpaper_panels__same_pattern_as_reference_label", + "seed": 3602 + }, + { + "domain": "illustrations", + "scene_id": "isometric_farmstead", + "task_id": "task_illustrations__isometric_farmstead__terrain_elevation_extremum_label", + "seed": 3701 + }, + { + "domain": "illustrations", + "scene_id": "pixel_village", + "task_id": "task_illustrations__pixel_village__territory_object_count", + "seed": 3702 + }, + { + "domain": "symbolic", + "scene_id": "chemical_equation", + "task_id": "task_symbolic__chemical_equation__balanced_option_label", + "seed": 3801 + }, + { + "domain": "symbolic", + "scene_id": "clock", + "task_id": "task_symbolic__clock__full_time_readout", + "seed": 3802 + }, + { + "domain": "pages", + "scene_id": "map", + "task_id": "task_pages__map__destination_after_directions_label", + "seed": 3901 + }, + { + "domain": "pages", + "scene_id": "timeline", + "task_id": "task_pages__timeline__relative_position_event_label", + "seed": 3902 + }, + { + "domain": "physics", + "scene_id": "ray_optics", + "task_id": "task_physics__ray_optics__ray_bounce_count", + "seed": 4001 + }, + { + "domain": "physics", + "scene_id": "circuit_state_change", + "task_id": "task_physics__circuit_state_change__bulb_brightness_change_label", + "seed": 4002 + }, + { + "domain": "puzzles", + "scene_id": "sudoku", + "task_id": "task_puzzles__sudoku__marked_cell_value", + "seed": 4101 + }, + { + "domain": "puzzles", + "scene_id": "polyomino_assembly", + "task_id": "task_puzzles__polyomino_assembly__composition_result_label", + "seed": 4102 + }, + { + "domain": "three_d", + "scene_id": "room", + "task_id": "task_three_d__room__wall_object_camera_distance_label", + "seed": 4201 + }, + { + "domain": "three_d", + "scene_id": "carousel", + "task_id": "task_three_d__carousel__between_object_type_anchors_count", + "seed": 4202 + } + ] +} diff --git a/examples/huggingface_space/requirements.txt b/examples/huggingface_space/requirements.txt new file mode 100644 index 00000000..3d4ad38c --- /dev/null +++ b/examples/huggingface_space/requirements.txt @@ -0,0 +1,18 @@ +Pillow==12.2.0 +CairoSVG==2.9.0 +tqdm==4.68.4 +PyYAML==6.0.3 +numpy==2.2.6 +scipy==1.15.3 +networkx==3.4.2 +rfc8785==0.1.4 +blake3==1.0.9 +zstandard==0.25.0 +cairocffi==1.7.1 +cffi==2.1.0 +pycparser==3.0 +cssselect2==0.9.0 +tinycss2==1.5.1 +webencodings==0.5.1 +defusedxml==0.7.1 +trace-tasks @ git+https://github.com/maveryn/trace.git@bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7 diff --git a/examples/huggingface_space/trace_demo.py b/examples/huggingface_space/trace_demo.py new file mode 100644 index 00000000..fe6a1516 --- /dev/null +++ b/examples/huggingface_space/trace_demo.py @@ -0,0 +1,375 @@ +"""Framework-independent logic for the public Trace task explorer.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import json +import math +from numbers import Integral, Real +from pathlib import Path +import secrets +from typing import Any + +from PIL import Image + +from trace_tasks import generate_task, list_task_ids +from trace_tasks.core.annotation_sanitization import ( + sanitize_trace_payload_for_public_annotation, +) +from trace_tasks.core.reward_contracts import resolve_reward_contract +from trace_tasks.core.source_layout_policy import parse_public_task_id +from trace_tasks.core.taxonomy import ACTIVE_DOMAINS + +from overlay import render_annotation_overlay + +REPOSITORY_URL = "https://github.com/maveryn/trace" +DOCUMENTATION_URL = "https://maveryn.github.io/trace/" +DATASET_URL = "https://huggingface.co/datasets/maveryn/trace" +SPACE_URL = "https://huggingface.co/spaces/maveryn/trace" +COLAB_URL = ( + "https://colab.research.google.com/github/maveryn/trace/blob/main/" + "examples/notebooks/trace_quickstart.ipynb" +) +PINNED_REVISION = "bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7" +DEFAULT_TASK_ID = "task_geometry__graph_paper__polygon_area_value" +DEFAULT_DOMAIN = "geometry" +DEFAULT_SCENE_ID = "graph_paper" +DEFAULT_SEED = 42 +MAX_SEED = (1 << 53) - 1 +MAX_ATTEMPTS = 100 + + +@dataclass(frozen=True) +class Preset: + """One deterministic curated example.""" + + domain: str + scene_id: str + task_id: str + seed: int + + @property + def label(self) -> str: + objective = parse_public_task_id(self.task_id).objective_contract + return f"{self.domain} · {self.scene_id} · {objective} · seed {self.seed}" + + +@dataclass(frozen=True) +class TaskCatalog: + """Cascading domain, scene, and task choices.""" + + task_ids: tuple[str, ...] + domains: tuple[str, ...] + scenes_by_domain: dict[str, tuple[str, ...]] + tasks_by_scene: dict[tuple[str, str], tuple[str, ...]] + + def scenes(self, domain: str) -> tuple[str, ...]: + if domain not in self.scenes_by_domain: + raise ValueError(f"unknown domain: {domain!r}") + return self.scenes_by_domain[domain] + + def tasks(self, domain: str, scene_id: str) -> tuple[str, ...]: + key = (domain, scene_id) + if key not in self.tasks_by_scene: + raise ValueError(f"unknown scene: {domain}/{scene_id}") + return self.tasks_by_scene[key] + + +@dataclass(frozen=True) +class RandomSelection: + """One uniformly sampled registered task and browser-safe seed.""" + + domain: str + scene_id: str + task_id: str + seed: int + + +@dataclass(frozen=True) +class DemoResult: + """Serializable outputs shown by the Gradio wrapper.""" + + original_image: Image.Image + annotation_overlay: Image.Image + prompt: str + ground_truth: dict[str, Any] + reward_contract: dict[str, Any] + trace_summary: dict[str, Any] + public_trace: dict[str, Any] + reproduction: str + links_markdown: str + + +def build_catalog(task_ids: Sequence[str] | None = None) -> TaskCatalog: + """Build deterministic cascading choices from the installed registry.""" + + resolved_task_ids = tuple(task_ids if task_ids is not None else list_task_ids()) + if not resolved_task_ids: + raise ValueError("Trace registry is empty") + if len(set(resolved_task_ids)) != len(resolved_task_ids): + raise ValueError("Trace registry contains duplicate task ids") + + mutable_scenes: dict[str, set[str]] = {} + mutable_tasks: dict[tuple[str, str], list[str]] = {} + for task_id in sorted(resolved_task_ids): + parts = parse_public_task_id(task_id) + mutable_scenes.setdefault(parts.domain, set()).add(parts.scene_id) + mutable_tasks.setdefault((parts.domain, parts.scene_id), []).append(task_id) + + active = [domain for domain in ACTIVE_DOMAINS if domain in mutable_scenes] + extras = sorted(set(mutable_scenes).difference(active)) + domains = tuple([*active, *extras]) + scenes = { + domain: tuple(sorted(mutable_scenes[domain])) + for domain in domains + } + tasks = { + key: tuple(sorted(values)) + for key, values in sorted(mutable_tasks.items()) + } + return TaskCatalog( + task_ids=tuple(sorted(resolved_task_ids)), + domains=domains, + scenes_by_domain=scenes, + tasks_by_scene=tasks, + ) + + +def sample_random_selection( + catalog: TaskCatalog | None = None, +) -> RandomSelection: + """Sample uniformly from every registered task and choose a fresh seed.""" + + resolved_catalog = catalog or build_catalog() + task_id = secrets.choice(resolved_catalog.task_ids) + parts = parse_public_task_id(task_id) + return RandomSelection( + domain=parts.domain, + scene_id=parts.scene_id, + task_id=task_id, + seed=secrets.randbelow(MAX_SEED + 1), + ) + + +def load_presets(path: Path | None = None) -> tuple[Preset, ...]: + """Load curated deterministic examples bundled with the Space.""" + + preset_path = path or Path(__file__).with_name("presets.json") + payload = json.loads(preset_path.read_text(encoding="utf-8")) + if payload.get("schema_version") != "trace_space_presets_v1": + raise ValueError("unsupported Trace Space preset schema") + + presets: list[Preset] = [] + for raw in payload.get("presets", []): + preset = Preset( + domain=str(raw["domain"]), + scene_id=str(raw["scene_id"]), + task_id=str(raw["task_id"]), + seed=validate_seed(raw["seed"]), + ) + parts = parse_public_task_id(preset.task_id) + if (parts.domain, parts.scene_id) != (preset.domain, preset.scene_id): + raise ValueError(f"preset taxonomy mismatch: {preset.task_id}") + presets.append(preset) + if len(presets) != 22: + raise ValueError(f"expected 22 curated presets, found {len(presets)}") + return tuple(presets) + + +def validate_seed(value: Any) -> int: + """Return a browser-safe integer seed.""" + + if isinstance(value, bool) or value is None: + raise ValueError("seed must be an integer") + if isinstance(value, Integral): + seed = int(value) + elif isinstance(value, Real) and math.isfinite(float(value)): + if not float(value).is_integer(): + raise ValueError("seed must be an integer") + seed = int(value) + elif isinstance(value, str): + normalized = value.strip() + if not normalized or not normalized.isdecimal(): + raise ValueError("seed must be an integer") + seed = int(normalized) + else: + raise ValueError("seed must be an integer") + if seed < 0 or seed > MAX_SEED: + raise ValueError(f"seed must be between 0 and {MAX_SEED}") + return seed + + +def generate_demo( + task_id: str, + seed: Any, + *, + catalog: TaskCatalog | None = None, +) -> DemoResult: + """Generate one deterministic task and its public inspection payloads.""" + + resolved_catalog = catalog or build_catalog() + normalized_task_id = str(task_id).strip() + if normalized_task_id not in set(resolved_catalog.task_ids): + raise ValueError("choose a registered Trace task") + normalized_seed = validate_seed(seed) + + output = generate_task( + normalized_task_id, + seed=normalized_seed, + params={}, + max_attempts=MAX_ATTEMPTS, + ) + answer_gt = json_safe(output.answer_gt.to_dict()) + annotation_gt = json_safe(output.annotation_gt.to_dict()) + reward_contract = resolve_reward_contract( + answer_type=output.answer_gt.type, + annotation_type=output.annotation_gt.type, + ).to_dict() + public_trace = sanitize_trace_payload_for_public_annotation( + output.trace_payload, + annotation_gt=output.annotation_gt, + ) + public_trace = json_safe(public_trace) + overlay = render_annotation_overlay(output.image, annotation_gt) + parts = parse_public_task_id(normalized_task_id) + + query_spec = public_trace.get("query_spec", {}) + prompt_trace = query_spec if isinstance(query_spec, Mapping) else {} + trace_summary = { + "task_id": normalized_task_id, + "taxonomy": { + "domain": parts.domain, + "scene_id": parts.scene_id, + "objective_contract": parts.objective_contract, + }, + "instance_seed": normalized_seed, + "resolved_scene_id": output.scene_id, + "query_id": output.query_id, + "image": { + "image_id": output.image_id, + "width": output.image.width, + "height": output.image.height, + }, + "answer_type": output.answer_gt.type, + "annotation_type": output.annotation_gt.type, + "prompt_selection": { + key: json_safe(prompt_trace[key]) + for key in ( + "template_id", + "prompt_variant", + "prompt_variant_active_key", + ) + if key in prompt_trace + }, + "task_versions": json_safe(output.task_versions), + "trace_sections": sorted(public_trace), + } + + source_path = ( + f"src/trace_tasks/tasks/{parts.domain}/{parts.scene_id}/" + f"{parts.objective_contract}.py" + ) + doc_path = f"docs/tasks/{parts.domain}/{parts.scene_id}/{normalized_task_id}.md" + source_url = f"{REPOSITORY_URL}/blob/{PINNED_REVISION}/{source_path}" + task_doc_url = f"{REPOSITORY_URL}/blob/{PINNED_REVISION}/{doc_path}" + answer_preview = json.dumps( + answer_gt["value"], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).replace("`", "'") + if len(answer_preview) > 120: + answer_preview = f"{answer_preview[:117]}..." + links = ( + f"**Typed result** · answer `{answer_gt['type']}` = `{answer_preview}` · " + f"annotation `{annotation_gt['type']}`\n\n" + f"**Verifier** · `{reward_contract['answer']['id']}` + " + f"`{reward_contract['annotation']['id']}`\n\n" + f"Generated from [`{normalized_task_id}`]({task_doc_url}) at " + f"[revision `{PINNED_REVISION[:7]}`]({source_url}). " + f"[Documentation]({DOCUMENTATION_URL}) · " + f"[Dataset]({DATASET_URL}) · [Colab]({COLAB_URL})" + ) + + reproduction = "\n".join( + [ + "python -m pip install \\", + ' "trace-tasks @ git+https://github.com/maveryn/trace.git' + f'@{PINNED_REVISION}"', + "", + "python - <<'PY'", + "from trace_tasks import generate_task", + "", + f'task_id = "{normalized_task_id}"', + f"sample = generate_task(task_id, seed={normalized_seed}, max_attempts=100)", + "sample.image.save('trace-example.png')", + "print(sample.prompt)", + "print(sample.answer_gt.to_dict())", + "print(sample.annotation_gt.to_dict())", + "PY", + ] + ) + + return DemoResult( + original_image=output.image.convert("RGB"), + annotation_overlay=overlay, + prompt=output.prompt, + ground_truth={ + "answer_gt": answer_gt, + "annotation_gt": annotation_gt, + }, + reward_contract=json_safe(reward_contract), + trace_summary=json_safe(trace_summary), + public_trace=public_trace, + reproduction=reproduction, + links_markdown=links, + ) + + +def json_safe(value: Any) -> Any: + """Convert Trace payload values to strict JSON-compatible objects.""" + + if value is None or isinstance(value, (str, bool)): + return value + if isinstance(value, Integral): + return int(value) + if isinstance(value, Real): + number = float(value) + if math.isfinite(number): + return number + return str(number) + if isinstance(value, Mapping): + return {str(key): json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + if isinstance(value, (set, frozenset)): + return [json_safe(item) for item in sorted(value, key=str)] + if hasattr(value, "to_dict"): + return json_safe(value.to_dict()) + if hasattr(value, "tolist"): + return json_safe(value.tolist()) + if hasattr(value, "item"): + return json_safe(value.item()) + return str(value) + + +__all__ = [ + "COLAB_URL", + "DEFAULT_DOMAIN", + "DEFAULT_SCENE_ID", + "DEFAULT_SEED", + "DEFAULT_TASK_ID", + "DemoResult", + "MAX_ATTEMPTS", + "MAX_SEED", + "PINNED_REVISION", + "Preset", + "SPACE_URL", + "TaskCatalog", + "build_catalog", + "generate_demo", + "json_safe", + "load_presets", + "validate_seed", +] diff --git a/examples/notebooks/trace_quickstart.ipynb b/examples/notebooks/trace_quickstart.ipynb new file mode 100644 index 00000000..2dd1d2c4 --- /dev/null +++ b/examples/notebooks/trace_quickstart.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Trace quickstart\n", + "\n", + "Generate a deterministic grounded visual-reasoning task, inspect its typed supervision and public execution trace, then replay its reward contract. No credentials or accelerator are required.\n", + "\n", + "[Open the Trace Space](https://huggingface.co/spaces/maveryn/trace) · [Read the documentation](https://maveryn.github.io/trace/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q \\\n", + " \"Pillow==12.2.0\" \"CairoSVG==2.9.0\" \"tqdm==4.68.4\" \"PyYAML==6.0.3\" \\\n", + " \"numpy==2.2.6\" \"scipy==1.15.3\" \"networkx==3.4.2\" \"rfc8785==0.1.4\" \\\n", + " \"blake3==1.0.9\" \"zstandard==0.25.0\" \"cairocffi==1.7.1\" \"cffi==2.1.0\" \\\n", + " \"pycparser==3.0\" \"cssselect2==0.9.0\" \"tinycss2==1.5.1\" \\\n", + " \"webencodings==0.5.1\" \"defusedxml==0.7.1\" \\\n", + " \"trace-tasks @ git+https://github.com/maveryn/trace.git@bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from IPython.display import JSON, display\n", + "from trace_tasks import generate_task, score_trace_response\n", + "from trace_tasks.core.annotation_sanitization import sanitize_trace_payload_for_public_annotation\n", + "from trace_tasks.core.reward_contracts import resolve_reward_contract\n", + "\n", + "TASK_ID = \"task_geometry__graph_paper__polygon_area_value\"\n", + "SEED = 42" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sample = generate_task(TASK_ID, seed=SEED, max_attempts=100)\n", + "display(sample.image)\n", + "print(sample.prompt)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "answer_gt = sample.answer_gt.to_dict()\n", + "annotation_gt = sample.annotation_gt.to_dict()\n", + "reward_contract = resolve_reward_contract(\n", + " answer_type=sample.answer_gt.type,\n", + " annotation_type=sample.annotation_gt.type,\n", + ").to_dict()\n", + "\n", + "display(JSON({\n", + " \"answer_gt\": answer_gt,\n", + " \"annotation_gt\": annotation_gt,\n", + " \"reward_contract\": reward_contract,\n", + "}))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "public_trace = sanitize_trace_payload_for_public_annotation(\n", + " sample.trace_payload,\n", + " annotation_gt=sample.annotation_gt,\n", + ")\n", + "display(JSON(public_trace))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "correct_response = json.dumps({\n", + " \"answer\": sample.answer_gt.value,\n", + " \"annotation\": sample.annotation_gt.value,\n", + "})\n", + "scores = score_trace_response(\n", + " response=correct_response,\n", + " answer_gt=answer_gt,\n", + " annotation_gt=annotation_gt,\n", + " reward_contract=reward_contract,\n", + " image_size=sample.image.size,\n", + ")\n", + "display(JSON(scores))\n", + "assert scores[\"overall\"] == 1.0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Try another task\n", + "\n", + "Change `TASK_ID` to any value returned by `trace_tasks.list_task_ids()` and choose any non-negative integer `SEED`. The same pair will always reproduce the same validated instance at the pinned code revision." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from trace_tasks import list_task_ids\n", + "\n", + "print(f\"{len(list_task_ids()):,} registered tasks\")\n", + "print(\"First five:\", *list_task_ids()[:5], sep=\"\\n- \")" + ] + } + ], + "metadata": { + "colab": { + "name": "Trace quickstart", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scripts/check_public_release.py b/scripts/check_public_release.py index 3692457e..2b4de6db 100644 --- a/scripts/check_public_release.py +++ b/scripts/check_public_release.py @@ -759,6 +759,20 @@ def _parse_json_output( ) from exc +def _venv_executable( + venv_root: Path, + name: str, + *, + os_name: str | None = None, +) -> Path: + """Return an installed virtual-environment command on POSIX or Windows.""" + + resolved_os_name = os.name if os_name is None else os_name + if resolved_os_name == "nt": + return venv_root / "Scripts" / f"{name}.exe" + return venv_root / "bin" / name + + def check_installed_cli( wheel_path: Path, workspace: Path, @@ -767,8 +781,8 @@ def check_installed_cli( ) -> None: venv_root = workspace / "venv" venv.EnvBuilder(with_pip=True, clear=True).create(venv_root) - bin_dir = venv_root / "bin" - pip = bin_dir / "python" + pip = _venv_executable(venv_root, "python") + bin_dir = pip.parent install_command: list[str | os.PathLike[str]] = [ pip, "-m", @@ -793,7 +807,11 @@ def check_installed_cli( _run(install_command, cwd=smoke_root, env=clean_env) listed = _parse_json_output( - _run([bin_dir / "trace-list", "--json"], cwd=smoke_root, env=clean_env), + _run( + [_venv_executable(venv_root, "trace-list"), "--json"], + cwd=smoke_root, + env=clean_env, + ), "trace-list", ) if ( @@ -807,7 +825,7 @@ def check_installed_cli( generated = _parse_json_output( _run( [ - bin_dir / "trace-generate", + _venv_executable(venv_root, "trace-generate"), "--task", REPRESENTATIVE_TASKS["geometry"], "--samples-per-task", @@ -832,7 +850,11 @@ def check_installed_cli( validation = _parse_json_output( _run( - [bin_dir / "trace-validate", dataset_root, "--json"], + [ + _venv_executable(venv_root, "trace-validate"), + dataset_root, + "--json", + ], cwd=smoke_root, env=clean_env, ), @@ -845,7 +867,7 @@ def check_installed_cli( exported = _parse_json_output( _run( [ - bin_dir / "trace-export", + _venv_executable(venv_root, "trace-export"), dataset_root, "--output", export_path, diff --git a/tests/test_huggingface_space.py b/tests/test_huggingface_space.py new file mode 100644 index 00000000..afcb2846 --- /dev/null +++ b/tests/test_huggingface_space.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +from collections import Counter +import json +from pathlib import Path +import re +import sys + +from PIL import Image +import pytest +import yaml + +from trace_tasks.core.annotation_sanitization import PUBLIC_IMAGE_ANNOTATION_TYPES + +REPO_ROOT = Path(__file__).resolve().parents[1] +SPACE_ROOT = REPO_ROOT / "examples" / "huggingface_space" +if str(SPACE_ROOT) not in sys.path: + sys.path.insert(0, str(SPACE_ROOT)) + +from overlay import PUBLIC_ANNOTATION_TYPES, render_annotation_overlay # noqa: E402 +from trace_demo import ( # noqa: E402 + DEFAULT_TASK_ID, + MAX_SEED, + PINNED_REVISION, + build_catalog, + generate_demo, + load_presets, + sample_random_selection, + validate_seed, +) + + +def test_space_catalog_exposes_the_complete_public_taxonomy() -> None: + catalog = build_catalog() + + assert len(catalog.task_ids) == 1_000 + assert len(catalog.domains) == 11 + assert sum(len(scenes) for scenes in catalog.scenes_by_domain.values()) == 277 + assert DEFAULT_TASK_ID in catalog.tasks("geometry", "graph_paper") + + +def test_space_random_selection_samples_the_complete_registry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog = build_catalog() + chosen_task_id = catalog.task_ids[-1] + calls: dict[str, object] = {} + + def choose(options: tuple[str, ...]) -> str: + calls["options"] = options + return chosen_task_id + + def randbelow(upper_bound: int) -> int: + calls["upper_bound"] = upper_bound + return upper_bound - 1 + + monkeypatch.setattr("trace_demo.secrets.choice", choose) + monkeypatch.setattr("trace_demo.secrets.randbelow", randbelow) + + selection = sample_random_selection(catalog) + + assert calls["options"] == catalog.task_ids + assert calls["upper_bound"] == MAX_SEED + 1 + assert selection.task_id == chosen_task_id + assert selection.task_id in catalog.tasks( + selection.domain, + selection.scene_id, + ) + assert selection.seed == MAX_SEED + + +def test_space_presets_match_the_release_gallery() -> None: + presets = load_presets() + gallery = json.loads( + (REPO_ROOT / "docs" / "gallery" / "manifest.v1.json").read_text( + encoding="utf-8" + ) + ) + expected = [ + { + "domain": item["domain"], + "scene_id": item["scene_id"], + "task_id": item["task_id"], + "seed": item["seed"], + } + for item in gallery["examples"] + ] + actual = [ + { + "domain": item.domain, + "scene_id": item.scene_id, + "task_id": item.task_id, + "seed": item.seed, + } + for item in presets + ] + + assert actual == expected + assert Counter(item.domain for item in presets) == { + domain: 2 for domain in build_catalog().domains + } + + +@pytest.mark.parametrize( + ("value", "expected"), + [(0, 0), (42, 42), (42.0, 42), ("42", 42), (MAX_SEED, MAX_SEED)], +) +def test_space_accepts_browser_safe_integer_seeds(value: object, expected: int) -> None: + assert validate_seed(value) == expected + + +@pytest.mark.parametrize( + "value", + [None, True, -1, MAX_SEED + 1, 1.5, "1.5", "", object()], +) +def test_space_rejects_invalid_seeds(value: object) -> None: + with pytest.raises(ValueError, match="seed"): + validate_seed(value) + + +_ANNOTATION_VALUES = { + "bbox": [10, 10, 50, 50], + "bbox_sequence": [[10, 10, 50, 50], [60, 60, 90, 90]], + "bbox_set": [[10, 10, 50, 50], [60, 60, 90, 90]], + "bbox_map": {"source": [10, 10, 50, 50]}, + "bbox_set_map": {"targets": [[10, 10, 50, 50], [60, 60, 90, 90]]}, + "point": [25, 25], + "point_sequence": [[25, 25], [75, 75]], + "point_set": [[25, 25], [75, 75]], + "point_map": {"source": [25, 25]}, + "point_set_map": {"targets": [[25, 25], [75, 75]]}, + "segment": [[10, 10], [90, 90]], + "segment_set": [[[10, 10], [90, 90]], [[90, 10], [10, 90]]], +} + + +@pytest.mark.parametrize("annotation_type", sorted(_ANNOTATION_VALUES)) +def test_space_overlay_renders_every_public_annotation_type( + annotation_type: str, +) -> None: + source = Image.new("RGB", (100, 100), "white") + overlay = render_annotation_overlay( + source, + {"type": annotation_type, "value": _ANNOTATION_VALUES[annotation_type]}, + ) + + assert overlay.mode == "RGB" + assert overlay.size == source.size + assert overlay.tobytes() != source.tobytes() + + +def test_space_overlay_type_set_stays_synced_with_public_abi() -> None: + assert PUBLIC_ANNOTATION_TYPES == PUBLIC_IMAGE_ANNOTATION_TYPES + + +def test_space_default_generation_exposes_public_contracts() -> None: + result = generate_demo(DEFAULT_TASK_ID, 42, catalog=build_catalog()) + + assert result.prompt + assert result.original_image.size == result.annotation_overlay.size + assert result.ground_truth["answer_gt"]["type"] + assert result.ground_truth["annotation_gt"]["type"] + assert result.reward_contract["reward_contract_version"] == "v0" + assert result.trace_summary["instance_seed"] == 42 + assert result.public_trace["witness_symbolic"] == { + "type": result.ground_truth["annotation_gt"]["type"], + "count": len(result.ground_truth["annotation_gt"]["value"]), + } + assert "**Typed result**" in result.links_markdown + assert result.reward_contract["answer"]["id"] in result.links_markdown + assert result.reward_contract["annotation"]["id"] in result.links_markdown + assert PINNED_REVISION in result.reproduction + assert "hf_" not in json.dumps(result.public_trace) + + +def test_space_card_and_runtime_are_pinned_for_public_cpu_deployment() -> None: + readme = (SPACE_ROOT / "README.md").read_text(encoding="utf-8") + match = re.match(r"^---\n(.*?)\n---\n", readme, flags=re.DOTALL) + assert match is not None + metadata = yaml.safe_load(match.group(1)) + + assert metadata["sdk"] == "gradio" + assert metadata["sdk_version"] == "6.20.0" + assert metadata["python_version"] == "3.12" + assert metadata["app_file"] == "app.py" + assert metadata["suggested_hardware"] == "cpu-basic" + assert metadata["colorFrom"] in { + "red", + "yellow", + "green", + "blue", + "indigo", + "purple", + "pink", + "gray", + } + assert metadata["colorTo"] in { + "red", + "yellow", + "green", + "blue", + "indigo", + "purple", + "pink", + "gray", + } + assert len(metadata["short_description"]) <= 60 + assert metadata["datasets"] == ["maveryn/trace"] + assert metadata["models"] == [ + "maveryn/trace-qwen2.5-vl-3b", + "maveryn/trace-qwen2.5-vl-7b", + ] + + requirements = (SPACE_ROOT / "requirements.txt").read_text(encoding="utf-8") + assert requirements.strip().endswith(f"@{PINNED_REVISION}") + release_constraints = { + line.split("==", 1)[0].lower(): line + for line in ( + REPO_ROOT / "constraints" / "release.txt" + ).read_text(encoding="utf-8").splitlines() + if "==" in line + } + space_pins = [ + line + for line in requirements.splitlines() + if "==" in line + ] + assert len(space_pins) == 17 + for requirement in space_pins: + package = requirement.split("==", 1)[0].lower() + assert release_constraints[package] == requirement + assert (SPACE_ROOT / "packages.txt").read_text(encoding="utf-8").strip() == ( + "libcairo2" + ) + + +def test_space_has_no_public_prediction_api_or_upload_input() -> None: + source = (SPACE_ROOT / "app.py").read_text(encoding="utf-8") + + assert "max_size=32" in source + assert "default_concurrency_limit=1" in source + assert "footer_links=[]" in source + assert source.count("api_name=False") == 5 + assert "gr.File(" not in source + assert "sources=" not in source + assert 'gr.Button("Random question"' in source + assert "_random_question" in source + assert "Random seed" not in source + assert "domain.input(" in source + assert "scene_id.input(" in source + + +def test_space_summary_only_shows_catalog_counts() -> None: + source = (SPACE_ROOT / "app.py").read_text(encoding="utf-8") + + assert source.count("class='trace-stat'") == 3 + assert "1,000tasks" in source + assert "277scenes" in source + assert "11domains" in source + assert "1 seed" not in source diff --git a/tests/test_public_release_check.py b/tests/test_public_release_check.py index 89ab57cd..1edce82b 100644 --- a/tests/test_public_release_check.py +++ b/tests/test_public_release_check.py @@ -408,3 +408,18 @@ def test_default_constraints_are_required(tmp_path: Path) -> None: match="default constraints file does not exist", ): release_check._resolve_constraints(tmp_path, None) + + +def test_virtual_environment_commands_are_portable(tmp_path: Path) -> None: + venv_root = tmp_path / "venv" + + assert release_check._venv_executable( + venv_root, + "python", + os_name="posix", + ) == venv_root / "bin" / "python" + assert release_check._venv_executable( + venv_root, + "trace-list", + os_name="nt", + ) == venv_root / "Scripts" / "trace-list.exe" diff --git a/tests/test_trace_quickstart_notebook.py b/tests/test_trace_quickstart_notebook.py new file mode 100644 index 00000000..b5501606 --- /dev/null +++ b/tests/test_trace_quickstart_notebook.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import json +from pathlib import Path +import re + +REPO_ROOT = Path(__file__).resolve().parents[1] +NOTEBOOK_PATH = ( + REPO_ROOT / "examples" / "notebooks" / "trace_quickstart.ipynb" +) +PINNED_REVISION = "bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7" +SECRET_RE = re.compile( + r"(?:hf_[A-Za-z0-9]{20,}|" + r"gh[pousr]_[A-Za-z0-9]{30,}|" + r"sk-[A-Za-z0-9_-]{20,})" +) + + +def _notebook() -> dict[str, object]: + return json.loads(NOTEBOOK_PATH.read_text(encoding="utf-8")) + + +def test_quickstart_notebook_is_valid_and_output_free() -> None: + notebook = _notebook() + + assert notebook["nbformat"] == 4 + assert notebook["nbformat_minor"] >= 5 + cells = notebook["cells"] + assert isinstance(cells, list) + assert cells + for cell in cells: + assert isinstance(cell, dict) + if cell["cell_type"] != "code": + continue + assert cell["execution_count"] is None + assert cell["outputs"] == [] + source = "".join(cell["source"]) + if source.startswith("%"): + continue + compile(source, str(NOTEBOOK_PATH), "exec") + + +def test_quickstart_notebook_pins_code_and_covers_the_public_workflow() -> None: + text = NOTEBOOK_PATH.read_text(encoding="utf-8") + + assert PINNED_REVISION in text + assert "task_geometry__graph_paper__polygon_area_value" in text + assert "generate_task" in text + assert "sanitize_trace_payload_for_public_annotation" in text + assert "resolve_reward_contract" in text + assert "score_trace_response" in text + assert 'scores[\\\"overall\\\"] == 1.0' in text + assert "list_task_ids" in text + + +def test_quickstart_notebook_contains_no_credentials() -> None: + text = NOTEBOOK_PATH.read_text(encoding="utf-8") + + assert SECRET_RE.search(text) is None + assert "hf-token" not in text.lower() diff --git a/tests/test_visibility_surfaces.py b/tests/test_visibility_surfaces.py new file mode 100644 index 00000000..bc6e2dbb --- /dev/null +++ b/tests/test_visibility_surfaces.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCREENSHOT = ( + REPO_ROOT / "docs" / "assets" / "examples" / "trace-space-quickstart.png" +) +SPACE_URL = "https://huggingface.co/spaces/maveryn/trace" +COLAB_URL = ( + "https://colab.research.google.com/github/maveryn/trace/blob/main/" + "examples/notebooks/trace_quickstart.ipynb" +) +PINNED_REVISION = "bb7fdd1fc8a0f8a2e3db7efe910a14e81d58feb7" + + +def test_landing_quickstarts_follow_the_domain_montage() -> None: + pages = { + "README.md": ( + "docs/assets/paper-domain-montage/trace-paper-domain-montage.png", + "## How Trace Works", + ), + "docs/README.md": ( + "assets/paper-domain-montage/trace-paper-domain-montage.png", + "## How Trace works", + ), + } + + for relative_path, (montage, architecture_heading) in pages.items(): + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + assert text.index(montage) < text.index("## Try Trace") + assert text.index("## Try Trace") < text.index(architecture_heading) + + +def test_live_demo_link_is_above_the_domain_montage() -> None: + pages = { + "README.md": ( + "docs/assets/paper-domain-montage/trace-paper-domain-montage.png" + ), + "docs/README.md": ( + "assets/paper-domain-montage/trace-paper-domain-montage.png" + ), + } + + for relative_path, montage in pages.items(): + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + assert text.index(SPACE_URL) < text.index(montage) + + +def test_landing_quickstarts_link_all_runnable_surfaces() -> None: + for relative_path in ("README.md", "docs/README.md"): + text = (REPO_ROOT / relative_path).read_text(encoding="utf-8") + assert SPACE_URL in text + assert COLAB_URL in text + assert PINNED_REVISION in text + assert "task_geometry__graph_paper__polygon_area_value" in text + assert "seed=42" in text + assert "trace-space-quickstart.png" in text + assert "researchers and engineers" in text + + +def test_landing_quickstart_screenshot_is_a_png() -> None: + assert SCREENSHOT.read_bytes().startswith(b"\x89PNG\r\n\x1a\n")