Add a command to dev.py to parse the pytest report

Author: dangusev

.github/workflows/daily.yml

@@ -3,23 +3,22 @@ on:
   schedule:
     # Runs at 03:00 UTC daily — adjust as needed
     - cron: '0 3 * * *'
-  workflow_dispatch: {}
+  workflow_dispatch: { }
 
 jobs:
   integration:
     uses: ./.github/workflows/run_tests.yml
     with:
       marker: 'integration'
+      additional_params: "--json-report --json-report-file=.report.json"
     secrets: inherit
 
   notify-slack:
-    needs: [integration]
+    needs: [ integration ]
     if: ${{ always() && needs.integration.result == 'failure' }}
     uses: ./.github/workflows/notify_slack.yml
     with:
       workflow-name: ${{ github.workflow }}
+      report-file: ".report.json"
       summary: Daily integration workflow failed.
-      details: |
-        Marker: integration
-        Result: integration=${{ needs.integration.result }}
     secrets: inherit

.github/workflows/notify_slack.yml

@@ -6,15 +6,14 @@ on:
         description: 'Display name of the workflow that failed.'
         required: true
         type: string
+      report-file:
+        description: 'JSON file with the test report.'
+        required: true
+        type: string
       summary:
         description: 'Short summary for the Slack notification.'
         required: true
         type: string
-      details:
-        description: 'Additional details to include in the Slack notification.'
-        required: false
-        default: ''
-        type: string
     secrets:
       SLACK_WEBHOOK_URL:
         required: true
@@ -24,6 +23,8 @@ jobs:
     name: Notify Slack on failure
     runs-on: ubuntu-latest
     steps:
+      - name: Generate pytest summary from JSON report
+        run: uv run --frozen dev.py parse-pytest-report --file .report.json > details.md
       - name: Post failure notification
         uses: slackapi/slack-github-action@v2.1.1
         with:
@@ -39,4 +40,4 @@ jobs:
               - type: section
                 text:
                   type: mrkdwn
-                  text: "*Details*\n${{ inputs.details }}"
+                  text: "*Details*\n$(cat details.md)""

.github/workflows/run_tests.yml

@@ -6,6 +6,11 @@ on:
         description: 'pytest -m expression (e.g., `not integration` or `integration`)'
         required: true
         type: string
+      pytest_params:
+        description: 'additional pytest params'
+        required: false
+        default: ""
+        type: string
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -89,6 +94,6 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/python-uv-setup
       - name: Run core tests
-        run: uv run pytest -m "${{ inputs.marker }}" tests/
+        run: uv run pytest -m "${{ inputs.marker }}" tests/ ${{ pytest_params }}
       - name: Run plugin tests
-        run: uv run pytest -m "${{ inputs.marker }}" plugins/*/tests/*.py
+        run: uv run pytest -m "${{ inputs.marker }}" plugins/*/tests/*.py ${{ pytest_params }}

.gitignore

@@ -96,3 +96,6 @@ profile.html
 .claude/
 
 .uv-cache/
+
+# pytest json report
+.report.json

agents-core/pyproject.toml

@@ -39,7 +39,7 @@ Website = "https://visionagents.ai/"
 Source = "https://github.com/GetStream/Vision-Agents"
 
 [project.optional-dependencies]
-dev = ["pytest", "mypy", "ruff", "click"]
+dev = ["pytest", "pytest-json-report", "mypy", "ruff", "click"]
 anthropic = ["vision-agents-plugins-anthropic"]
 cartesia = ["vision-agents-plugins-cartesia"]
 deepgram = ["vision-agents-plugins-deepgram"]

dev.py

@@ -4,6 +4,8 @@
 Essential dev commands for testing, linting, and type checking
 """
 
+import datetime
+import json
 import os
 import shlex
 import subprocess
@@ -216,5 +218,92 @@ def check():
     click.echo("\n✅ All checks passed!")
 
 
+def _extract_first_failure(
+    tests: list[dict],
+) -> tuple[str, str, str] | tuple[None, None, None]:
+    """
+    Return (nodeid, phase, message) for first failed test,
+    checking setup → call → teardown.
+    """
+    for t in tests:
+        if t.get("outcome") in ("failed", "error"):
+            nodeid = t.get("nodeid", "unknown test")
+
+            for phase in ["setup", "call", "teardown"]:
+                phase_data = t.get(phase)
+                if phase_data and phase_data.get("outcome") == "failed":
+                    longrepr = phase_data.get("longrepr", "")
+                    if isinstance(longrepr, dict):
+                        message = longrepr.get("reprcrash", {}).get(
+                            "message", str(longrepr)
+                        )
+                    else:
+                        message = str(longrepr)
+
+                    return nodeid, phase, message
+
+            return nodeid, "unknown", "No error details available"
+
+    return None, None, None
+
+
+@cli.command()
+@click.option(
+    "--file",
+    "report_file",
+    default=".report.json",
+    type=click.File("r"),
+    show_default=True,
+    help="Path to pytest JSON report file",
+)
+def parse_pytest_report(report_file):
+    """Parse pytest JSON report and print summary for CI (GitHub Actions friendly)."""
+    report = json.load(report_file)
+
+    summary = report.get("summary", {})
+    duration = str(datetime.timedelta(report.get("duration", 0) / 1000))
+    exit_code = report.get("exitcode", 0)
+
+    failed = summary.get("failed", 0)
+    error = summary.get("error", 0)
+    passed = summary.get("passed", 0)
+    skipped = summary.get("skipped", 0)
+    total = summary.get("total", 0)
+
+    # Header
+    status = (
+        "❌ Test Results - Failed" if failed or error else "✅ Test Results - Success"
+    )
+    click.echo(f"**{status}**")
+    click.echo()
+
+    # Duration
+    click.echo(f"**Duration:** {duration}")
+    click.echo()
+
+    # Summary
+    click.echo("**Summary:**")
+    click.echo(f"- Total: {total}")
+    click.echo(f"- Passed: {passed}")
+    click.echo(f"- Failed: {failed}")
+    click.echo(f"- Error: {error}")
+    click.echo(f"- Skipped: {skipped}")
+    click.echo()
+
+    # Failure details
+    if failed or error:
+        nodeid, phase, message = _extract_first_failure(report.get("tests", []))
+
+        click.echo("**Failure Details:**")
+        click.echo(f"- Exit code: `{exit_code}`")
+
+        if nodeid:
+            click.echo(f"- First failed test: `{nodeid}`")
+        if phase:
+            click.echo(f"- Phase: `{phase}`")
+        if message:
+            click.echo(f"- Error:\n\n```\n{message}\n```")
+
+
 if __name__ == "__main__":
     cli()