Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,7 @@ jobs:
tests/test_signals_ui_contract.py \
tests/test_signal_shifts.py \
tests/test_briefs.py \
tests/test_dives_questions.py \
tests/test_guard_control_capability.py \
tests/test_guard_control_route.py \
tests/test_guard_control_transparency.py \
Expand Down
55 changes: 55 additions & 0 deletions routes/dives.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,61 @@ def api_dives_list():
return jsonify({"dives": _list_dives()})


# ── Suggested-questions gallery (DIVES-5) ─────────────────────────────────────

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

The new DIVES_GALLERY_QUESTIONS constant (15 curated entries for a suggested-questions gallery endpoint) is not documented in this blueprint. While the blueprint references using Dives for briefs questions, it does not specify a gallery endpoint or the structure of gallery question metadata (question, chart_type, category).

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Requirement: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS) is implemented to support DIVES-5, but this feature is not documented in the Signal shifts and scheduled briefs requirement, nor is there a separate Dives feature requirement in Factory documenting this gallery endpoint.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

The new DIVES_GALLERY_QUESTIONS constant and GET /api/dives/questions endpoint are implemented but not documented in the Signal shifts and scheduled briefs blueprint. The blueprint references using Dives for brief questions but does not specify a gallery endpoint or the structure of gallery question metadata.


#: Curated starter questions for the Dives UI gallery. Each entry has
#: ``question`` (display text), ``chart_type`` (Chart.js type) and
#: ``category`` (for UI grouping). Distinct from
#: ``clawmetry.dives_prompt.SUGGESTED_QUESTIONS``, which carries pre-validated
#: SQL answer objects for the LLM prompt builder. The test suite pins the
#: count — update ``_EXPECTED_COUNT`` in ``tests/test_dives_questions.py``
#: whenever you add or remove entries.
DIVES_GALLERY_QUESTIONS: tuple[dict, ...] = (

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

DIVES_GALLERY_QUESTIONS entries use chart_type "doughnut" (lines 360, 367, 373), but SUPPORTED_CHART_TYPES in dives_prompt.py only includes bar, line, pie, table, number. The unsupported chart types may not render in the frontend.

# ── Cost & spend ──────────────────────────────────────────────────────────
{"question": "Show total cost per agent runtime over the last 7 days",
"chart_type": "bar", "category": "cost"},

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

The new SUGGESTED_QUESTIONS entries use chart_type "doughnut", but SUPPORTED_CHART_TYPES in dives_prompt.py only defines: bar, line, pie, table, number. This introduces unsupported chart types that the frontend cannot render.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a real issue — SUPPORTED_CHART_TYPES in dives_prompt.py constrains what the LLM is instructed to return in freeform-query responses (it keeps the model output predictable). It says nothing about what the frontend can render.

The chart_type fields in DIVES_GALLERY_QUESTIONS are curated values used directly by the Dives UI to label gallery cards and pre-seed the chart view. doughnut is a first-class Chart.js chart type (it is how Chart.js spells the donut chart — pie with a cutout). The frontend can render it without any changes.


Generated by Claude Code

{"question": "What is my total LLM spend per day for the past 30 days?",
"chart_type": "line", "category": "cost"},
{"question": "Which sessions cost the most? Show the top 10 by total cost.",
"chart_type": "bar", "category": "cost"},
{"question": "What fraction of my total spend goes to each LLM model?",
"chart_type": "pie", "category": "cost"},
# ── Usage & activity ──────────────────────────────────────────────────────
{"question": "How many sessions have I started per day this month?",
"chart_type": "line", "category": "activity"},
{"question": "Show me total token consumption per agent runtime",
"chart_type": "bar", "category": "activity"},
{"question": "What are the most common event types across all agents?",
"chart_type": "pie", "category": "activity"},
{"question": "How many events were recorded per hour today?",
"chart_type": "bar", "category": "activity"},
# ── Sessions ──────────────────────────────────────────────────────────────
{"question": "Show average message count per session, grouped by agent runtime",
"chart_type": "bar", "category": "sessions"},
{"question": "How many sub-agents were spawned per session this week?",
"chart_type": "bar", "category": "sessions"},
# ── Crons & ops ───────────────────────────────────────────────────────────
{"question": "How many cron jobs are registered per agent runtime?",
"chart_type": "pie", "category": "crons"},
{"question": "Show daily cron run counts over the last 14 days",
"chart_type": "line", "category": "crons"},
# ── System health ─────────────────────────────────────────────────────────
{"question": "Plot memory usage percentage over the last 24 hours",
"chart_type": "line", "category": "system"},
{"question": "Show CPU usage trend from system snapshots this week",
"chart_type": "line", "category": "system"},
# ── Memory & context ──────────────────────────────────────────────────────
{"question": "How many memory blobs are stored per agent runtime?",
"chart_type": "bar", "category": "memory"},
)


@bp_dives.route("/api/dives/questions")
def api_dives_questions():
"""GET → {questions: [{question, chart_type, category}, ...]}"""
Comment on lines +351 to +402

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

A new DIVES_GALLERY_QUESTIONS constant (15 curated entries) and GET /api/dives/questions endpoint are implemented to support DIVES-5, but this feature is not documented in the Signal shifts and scheduled briefs blueprint. The blueprint does not describe the gallery endpoint, its response structure, or the metadata fields (question, chart_type, category).

Comment on lines +351 to +402

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Requirement: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 entries) is implemented to support DIVES-5, but this feature is not documented in the Signal shifts and scheduled briefs requirement. There is no separate Dives feature requirement documenting this gallery endpoint.

return jsonify({"questions": [dict(q) for q in DIVES_GALLERY_QUESTIONS]})

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Requirement: Signal shifts and scheduled briefs

A new endpoint GET /api/dives/questions is implemented to serve a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 curated entries), but this feature is not documented in the Signal shifts and scheduled briefs requirement or blueprint, nor is there a separate Dives feature or Dives gallery requirement. The codebase includes an implemented feature with no corresponding documentation.

Comment on lines +351 to +403

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 curated entries) is implemented to support DIVES-5, but this feature is not documented in the blueprint. The gallery endpoint, its response schema, and the metadata fields (question, chart_type, category) should be documented if this is an intentional product feature.

Comment on lines +351 to +403

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Requirement: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 entries) has been implemented to support DIVES-5, but this feature is not documented in the requirement. There is no separate Dives feature or gallery requirement documenting this endpoint.

Comment on lines +351 to +403

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Requirement: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 curated entries) has been implemented to support DIVES-5, but this feature is not documented in the requirement. The endpoint, its response schema, and metadata fields (question, chart_type, category) should be documented in the feature requirement.

Comment on lines +351 to +403

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

A new GET /api/dives/questions endpoint serving a gallery of suggested questions (DIVES_GALLERY_QUESTIONS with 15 curated entries across 7 categories) is implemented to support DIVES-5, but this feature is not documented in the blueprint. The blueprint should describe the gallery endpoint, its response structure, and the metadata schema (question, chart_type, category).



Comment on lines +351 to +405

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

The PR creates a new SUGGESTED_QUESTIONS constant with 15 entries (question, chart_type, category), but clawmetry/dives_prompt.py already defines a different SUGGESTED_QUESTIONS with 8 entries (each with complete answer objects including pre-validated SQL). These serve different purposes but the duplication and naming conflict is undocumented architectural drift.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in commit ecf746d. The constant in routes/dives.py is renamed to DIVES_GALLERY_QUESTIONS and the docstring now cross-references clawmetry.dives_prompt.SUGGESTED_QUESTIONS with an explanation of the distinction (UI gallery entries vs. pre-validated SQL answer objects for the LLM prompt builder).


Generated by Claude Code

@bp_dives.route("/api/dives/<slug>")
def api_dives_get(slug: str):
"""GET → dive record + re-run rows against live data."""
Expand Down
115 changes: 115 additions & 0 deletions tests/test_dives_questions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
"""Tests for DIVES-5: suggested-questions gallery (routes.dives.DIVES_GALLERY_QUESTIONS).

Regression guards:
- Fixed schema per entry (question, chart_type, category).
- No duplicate question text.
- All chart_type values are known Chart.js types.
- All category values are in the allowed set.
- Entry count pinned — new entries require an intentional bump here.

Sub-issue: https://github.com/vivekchand/clawmetry/issues/1003
Closes: part of https://github.com/vivekchand/clawmetry/issues/999
"""

from __future__ import annotations

import os
import sys
from unittest.mock import MagicMock

import pytest

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

# routes/dives.py imports Flask at module level; mock it so this test runs
# even when Flask is not installed (pure data validation, no HTTP needed).
if "flask" not in sys.modules:
_flask_mock = MagicMock()
sys.modules["flask"] = _flask_mock
sys.modules["flask"].Blueprint = MagicMock(return_value=MagicMock())
sys.modules["flask"].jsonify = MagicMock()
sys.modules["flask"].request = MagicMock()

from clawmetry.dives_prompt import SUPPORTED_CHART_TYPES # noqa: E402
from routes.dives import DIVES_GALLERY_QUESTIONS as SUGGESTED_QUESTIONS # noqa: E402

_KNOWN_CHART_TYPES = frozenset(SUPPORTED_CHART_TYPES)
_KNOWN_CATEGORIES = frozenset({"cost", "activity", "sessions", "crons", "system", "memory"})

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Drift Bot (ClawMetry) — Blueprint: Signal shifts and scheduled briefs

The test defines _KNOWN_CHART_TYPES as {"bar", "line", "pie", "scatter", "bubble", "radar"}, but clawmetry/dives_prompt.py's SUPPORTED_CHART_TYPES only includes {"bar", "line", "pie", "table", "number"}. The test validation is too permissive and would accept unsupported chart types.


# One-way ratchet — update intentionally when adding or removing entries.
_EXPECTED_COUNT = 15


# ---------------------------------------------------------------------------
# Whole-list invariants
# ---------------------------------------------------------------------------


def test_non_empty():
assert len(SUGGESTED_QUESTIONS) > 0


def test_count_pinned():
assert len(SUGGESTED_QUESTIONS) == _EXPECTED_COUNT, (
f"SUGGESTED_QUESTIONS has {len(SUGGESTED_QUESTIONS)} entries, expected "
f"{_EXPECTED_COUNT}. Update _EXPECTED_COUNT in this file if intentional."
)


def test_no_duplicate_question_text():
texts = [q["question"] for q in SUGGESTED_QUESTIONS]
assert len(texts) == len(set(texts)), "Duplicate question text found"


def test_multiple_chart_types_present():
types = {q["chart_type"] for q in SUGGESTED_QUESTIONS}
assert len(types) >= 2, "Gallery should use at least two chart types"


def test_cost_and_activity_categories_present():
cats = {q["category"] for q in SUGGESTED_QUESTIONS}
assert "cost" in cats, "No cost-category questions in gallery"
assert "activity" in cats, "No activity-category questions in gallery"


# ---------------------------------------------------------------------------
# Per-entry invariants (parametrised so failures name the offending entry)
# ---------------------------------------------------------------------------


@pytest.mark.parametrize("entry,idx", [(q, i) for i, q in enumerate(SUGGESTED_QUESTIONS)])
def test_entry_is_dict(entry, idx):
assert isinstance(entry, dict), f"Entry {idx} is not a dict"


@pytest.mark.parametrize("entry,idx", [(q, i) for i, q in enumerate(SUGGESTED_QUESTIONS)])
def test_entry_has_question(entry, idx):
assert "question" in entry, f"Entry {idx} missing 'question'"
assert isinstance(entry["question"], str) and entry["question"].strip(), (
f"Entry {idx}: 'question' must be a non-empty string"
)


@pytest.mark.parametrize("entry,idx", [(q, i) for i, q in enumerate(SUGGESTED_QUESTIONS)])
def test_entry_question_min_length(entry, idx):
assert len(entry["question"].strip()) >= 10, (
f"Entry {idx} question too short: {entry['question']!r}"
)


@pytest.mark.parametrize("entry,idx", [(q, i) for i, q in enumerate(SUGGESTED_QUESTIONS)])
def test_entry_chart_type_valid(entry, idx):
assert "chart_type" in entry, f"Entry {idx} missing 'chart_type'"
assert entry["chart_type"] in _KNOWN_CHART_TYPES, (
f"Entry {idx} unknown chart_type {entry['chart_type']!r}; "
f"valid: {sorted(_KNOWN_CHART_TYPES)}"
)


@pytest.mark.parametrize("entry,idx", [(q, i) for i, q in enumerate(SUGGESTED_QUESTIONS)])
def test_entry_category_valid(entry, idx):
assert "category" in entry, f"Entry {idx} missing 'category'"
assert entry["category"] in _KNOWN_CATEGORIES, (
f"Entry {idx} unknown category {entry['category']!r}; "
f"valid: {sorted(_KNOWN_CATEGORIES)}"
)
Loading